code:
import os, socket, subprocess, threading, traceback

from lib.Hydrus.Data import GetNow, GetSubprocessKWArgs, Print, TimeHasPassed, \
	WaitForProcessToFinish
from lib.Hydrus.Exceptions import FirewallException
from lib.Hydrus.Text import DeserialiseNewlinedTexts

### CONSTANTS ###
import lib.Hydrus.Constants as HC

if HC.PLATFORM_LINUX:
	upnpc_path = os.path.join( HC.BIN_DIR, 'upnpc_linux' )

elif HC.PLATFORM_OSX:
	upnpc_path = os.path.join( HC.BIN_DIR, 'upnpc_osx' )

elif HC.PLATFORM_WINDOWS:
	upnpc_path = os.path.join( HC.BIN_DIR, 'upnpc_win32.exe' )

EXTERNAL_IP = {}
EXTERNAL_IP[ 'ip' ] = None
EXTERNAL_IP[ 'time' ] = 0

### FUNCTIONS ###

def GetExternalIP():
	if 'external_host' in HC.options and HC.options[ 'external_host' ] is not None:
		return HC.options[ 'external_host' ]

	if TimeHasPassed( EXTERNAL_IP[ 'time' ] + ( 3600 * 24 ) ):
		cmd = [ upnpc_path, '-l' ]

		sbp_kwargs = GetSubprocessKWArgs( text = True )

		p = subprocess.Popen( cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE, **sbp_kwargs )

		WaitForProcessToFinish( p, 30 )

		( output, error ) = p.communicate()

		if error is not None and len( error ) > 0:
			raise Exception( 'Problem while trying to fetch External IP:' + os.linesep * 2 + str( error ) )

		else:
			try:
				lines = DeserialiseNewlinedTexts( output )

				i = lines.index( 'i protocol exPort->inAddr:inPort description remoteHost leaseTime' )

				# ExternalIPAddress = ip
				( gumpf, external_ip_address ) = lines[ i - 1 ].split( ' = ' )

			except ValueError:
				raise Exception( 'Could not parse external IP!' )

			if external_ip_address == '0.0.0.0':
				raise Exception( 'Your UPnP device returned your external IP as 0.0.0.0! Try rebooting it, or overwrite it in options!' )

			EXTERNAL_IP[ 'ip' ] = external_ip_address
			EXTERNAL_IP[ 'time' ] = GetNow()

	return EXTERNAL_IP[ 'ip' ]

def GetLocalIP():
	return socket.gethostbyname( socket.gethostname() )

def AddUPnPMapping( internal_client, internal_port, external_port, protocol, description, duration = 3600 ):
	cmd = [ upnpc_path, '-e', description, '-a', internal_client, str( internal_port ), str( external_port ), protocol, str( duration ) ]

	sbp_kwargs = GetSubprocessKWArgs( text = True )

	p = subprocess.Popen( cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE, **sbp_kwargs )

	WaitForProcessToFinish( p, 30 )

	( output, error ) = p.communicate()

	if 'x.x.x.x:' + str( external_port ) + ' TCP is redirected to internal ' + internal_client + ':' + str( internal_port ) in output:
		raise FirewallException( 'The UPnP mapping of ' + internal_client + ':' + internal_port + '->external:' + external_port + ' already exists as a port forward. If this UPnP mapping is automatic, please disable it.' )

	if output is not None and 'failed with code' in output:
		if 'UnknownError' in output:
			raise FirewallException( 'Problem while trying to add UPnP mapping:' + os.linesep * 2 + output )

		else:
			raise Exception( 'Problem while trying to add UPnP mapping:' + os.linesep * 2 + output )

	if error is not None and len( error ) > 0:
		raise Exception( 'Problem while trying to add UPnP mapping:' + os.linesep * 2 + error )

def GetUPnPMappings():
	external_ip_address = GetExternalIP()

	cmd = [ upnpc_path, '-l' ]

	sbp_kwargs = GetSubprocessKWArgs( text = True )

	p = subprocess.Popen( cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE, **sbp_kwargs )

	WaitForProcessToFinish( p, 30 )

	( output, error ) = p.communicate()

	if error is not None and len( error ) > 0:
		raise Exception( 'Problem while trying to fetch UPnP mappings:' + os.linesep * 2 + error )

	else:
		try:
			lines = DeserialiseNewlinedTexts( output )

			i = lines.index( 'i protocol exPort->inAddr:inPort description remoteHost leaseTime' )

			data_lines = []

			i += 1

			while i < len( lines ):
				if not lines[ i ][0] in ( ' ', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' ):
					break

				data_lines.append( lines[ i ] )

				i += 1

			processed_data = []

			for line in data_lines:
				# 0 UDP 65533->192.168.0.197:65533 'Skype UDP at 192.168.0.197:65533 (2665)' '' 0

				if line.count( ':' ) != 1:
					continue

				while '  ' in line:
					line = line.replace( '  ', ' ' )

				if line.startswith( ' ' ):
					( empty, number, protocol, mapping_data, rest_of_line ) = line.split( ' ', 4 )

				else:
					( number, protocol, mapping_data, rest_of_line ) = line.split( ' ', 3 )

				( external_port, rest_of_mapping_data ) = mapping_data.split( '->' )

				external_port = int( external_port )

				if rest_of_mapping_data.count( ':' ) == 1:
					( internal_client, internal_port ) = rest_of_mapping_data.split( ':' )

				else:
					parts = rest_of_mapping_data.split( ':' )

					internal_port = parts.pop( -1 )

					internal_client = ':'.join( parts )

				internal_port = int( internal_port )

				( empty, description, space, remote_host, rest_of_line ) = rest_of_line.split( '\'', 4 )

				lease_time = int( rest_of_line[1:] )

				processed_data.append( ( description, internal_client, internal_port, external_ip_address, external_port, protocol, lease_time ) )

			return processed_data

		except Exception as e:
			Print( 'UPnP problem:' )
			Print( traceback.format_exc() )
			Print( 'Full response follows:' )
			Print( output )

			raise Exception( 'Problem while trying to parse UPnP mappings:' + os.linesep * 2 + str( e ) )

def RemoveUPnPMapping( external_port, protocol ):
	cmd = [ upnpc_path, '-d', str( external_port ), protocol ]

	sbp_kwargs = GetSubprocessKWArgs( text = True )

	p = subprocess.Popen( cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE, **sbp_kwargs )

	WaitForProcessToFinish( p, 30 )

	( output, error ) = p.communicate()

	if error is not None and len( error ) > 0:
		raise Exception( 'Problem while trying to remove UPnP mapping:' + os.linesep * 2 + error )

### CLASSES ###

class ServicesUPnPManager( object ):
	def __init__( self, services ):
		self._lock = threading.Lock()

		self._services = services

	def _RefreshUPnP( self, force_wipe = False ):
		if not force_wipe:
			running_service_with_upnp = True in ( service.GetPort() is not None and service.GetUPnPPort() is not None for service in self._services )

			if not running_service_with_upnp:
				return

		try:
			local_ip = GetLocalIP()

			current_mappings = GetUPnPMappings()

			our_mappings = { ( internal_client, internal_port ) : external_port for ( description, internal_client, internal_port, external_ip_address, external_port, protocol, enabled ) in current_mappings }

		except Exception:
			return # This IGD probably doesn't support UPnP, so don't spam the user with errors they can't fix!

		for service in self._services:
			internal_port = service.GetPort()
			upnp_port = service.GetUPnPPort()

			if ( local_ip, internal_port ) in our_mappings:
				current_external_port = our_mappings[ ( local_ip, internal_port ) ]

				port_is_incorrect = upnp_port is None or upnp_port != current_external_port

				if port_is_incorrect or force_wipe:
					RemoveUPnPMapping( current_external_port, 'TCP' )

		for service in self._services:
			internal_port = service.GetPort()
			upnp_port = service.GetUPnPPort()

			if upnp_port is not None:
				service_type = service.GetServiceType()

				protocol = 'TCP'

				description = HC.service_string_lookup[ service_type ] + ' at ' + local_ip + ':' + str( internal_port )

				duration = 86400

				try:
					AddUPnPMapping( local_ip, internal_port, upnp_port, protocol, description, duration = duration )

				except FirewallException:
					Print( 'The UPnP Daemon tried to add ' + local_ip + ':' + internal_port + '->external:' + upnp_port + ' but it failed due to router error. Please try it manually to get a full log of what happened.' )

					return

	def SetServices( self, services ):
		with self._lock:
			self._services = services

			self._RefreshUPnP( force_wipe = True )

	def RefreshUPnP( self ):
		with self._lock:
			self._RefreshUPnP()


code:
import calendar, collections, datetime, json, psutil, socket, threading, urllib, urllib3
from urllib3.exceptions import InsecureRequestWarning

from lib.Hydrus.Data import AccountIdentifier, ContentUpdate, ConvertTimestampToPrettyExpires, \
	ConvertValueRangeToBytes, ConvertValueRangeToPrettyString, GenerateKey, \
	GetNow, GetTimeDeltaUntilTime, TimeDeltaToPrettyTimeDelta, TimeHasPassed, \
	TimestampToPrettyTimeDelta, ToHumanBytes, ToHumanInt
from lib.Hydrus.Exceptions import BadRequestException, DataMissing, InsufficientCredentialsException
from lib.Hydrus.Serialisable import CreateFromNetworkBytes, \
	CreateFromSerialisableTuple, CreateFromString, SerialisableDictionary, SerialisableBase

### CONSTANTS ###
import lib.Hydrus.Constants as HC
import lib.Hydrus.Globals as HG

from lib.Hydrus.Serialisable import HG.SerialisableType["BANDWIDTH_RULES"], \
	HG.SerialisableType["BANDWIDTH_TRACKER"], HG.SerialisableType["CLIENT_TO_SERVER_UPDATE"], \
	HG.SerialisableType["CONTENT"], HG.SerialisableType["CONTENT_UPDATE"], HG.SerialisableType["CREDENTIALS"], \
	HG.SerialisableType["DEFINITIONS_UPDATE"], HG.SerialisableType["METADATA"], \
	HG.SerialisableType["PETITION"], SERIALISABLE_TYPES_TO_OBJECT_TYPES

INT_PARAMS = { 'expires', 'num', 'since', 'content_type', 'action', 'status' }
BYTE_PARAMS = { 'access_key', 'account_type_key', 'subject_account_key', 'hash', 'registration_key', 'subject_hash', 'update_hash' }
STRING_PARAMS = { 'subject_tag' }
JSON_PARAMS = set()

urllib3.disable_warnings( InsecureRequestWarning ) # stopping log-moaning when request sessions have verify = False

# The calendar portion of this works in GMT. A new 'day' or 'month' is calculated based on GMT time, so it won't tick over at midnight for most people.
# But this means a server can pass a bandwidth object to a lad and everyone can agree on when a new day is.

### FUNCTIONS ###

def ConvertBandwidthRuleToString( rule ):
	( bandwidth_type, time_delta, max_allowed ) = rule

	if max_allowed == 0:
		return 'No requests currently permitted.'

	if bandwidth_type == HC.BANDWIDTH_TYPE_DATA:
		s = ToHumanBytes( max_allowed )

	elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS:
		s = ToHumanInt( max_allowed ) + ' rqs'

	if time_delta is None:
		s += ' per month'

	else:
		s += ' per ' + TimeDeltaToPrettyTimeDelta( time_delta )

	return s

def LocalPortInUse( port ):
	if HC.PLATFORM_WINDOWS:
		for sconn in psutil.net_connections():
			if port == sconn.laddr[1] and sconn.status in ( 'ESTABLISHED', 'LISTEN' ): # local address: ( ip, port )
				return True

		return False

	else:
		s = socket.socket( socket.AF_INET, socket.SOCK_STREAM )

		s.settimeout( 0.2 )

		result = s.connect_ex( ( '127.0.0.1', port ) )

		s.close()

		CONNECTION_SUCCESS = 0

		return result == CONNECTION_SUCCESS

def ParseTwistedRequestGETArgs( requests_args, int_params, byte_params, string_params, json_params ):
	args = ParsedRequestArguments()

	for name_bytes in requests_args:
		values_bytes = requests_args[ name_bytes ]

		try:
			name = str( name_bytes, 'utf-8' )

		except UnicodeDecodeError:
			continue

		value_bytes = values_bytes[0]

		try:
			value = str( value_bytes, 'utf-8' )

		except UnicodeDecodeError:
			continue

		if name in int_params:
			try:
				args[ name ] = int( value )

			except Exception:
				raise BadRequestException( 'I was expecting to parse \'' + name + '\' as an integer, but it failed.' )

		elif name in byte_params:
			try:
				args[ name ] = bytes.fromhex( value )

			except Exception:
				raise BadRequestException( 'I was expecting to parse \'' + name + '\' as a hex-encoded bytestring, but it failed.' )

		elif name in string_params:
			try:
				args[ name ] = urllib.parse.unquote( value )

			except Exception:
				raise BadRequestException( 'I was expecting to parse \'' + name + '\' as a percent-encdode string, but it failed.' )

		elif name in json_params:
			try:
				args[ name ] = json.loads( urllib.parse.unquote( value ) )

			except Exception:
				raise BadRequestException( 'I was expecting to parse \'' + name + '\' as a json-encoded string, but it failed.' )

	return args

class ParsedRequestArguments( dict ):
	def __missing__( self, key ):

		raise BadRequestException( 'It looks like the parameter "{}" was missing!'.format( key ) )

class BandwidthRules( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["BANDWIDTH_RULES"]
	SERIALISABLE_NAME = 'Bandwidth Rules'
	SERIALISABLE_VERSION = 1

	def __init__( self ):
		SerialisableBase.__init__( self )

		self._lock = threading.Lock()

		self._rules = set()

	def _GetSerialisableInfo( self ):
		return list( self._rules )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		# tuples converted to lists via json

		self._rules = set( ( tuple( rule_list ) for rule_list in serialisable_info ) )

	def AddRule( self, bandwidth_type, time_delta, max_allowed ):
		with self._lock:
			rule = ( bandwidth_type, time_delta, max_allowed )

			self._rules.add( rule )

	def CanContinueDownload( self, bandwidth_tracker, threshold = 15 ):
		with self._lock:
			for ( bandwidth_type, time_delta, max_allowed ) in self._rules:
				# Do not stop ongoing just because starts are throttled
				requests_rule = bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS

				# Do not block an ongoing jpg download because the current month is 100.03% used
				wait_is_too_long = time_delta is None or time_delta > threshold

				ignore_rule = requests_rule or wait_is_too_long

				if ignore_rule:
					continue

				if bandwidth_tracker.GetUsage( bandwidth_type, time_delta ) >= max_allowed:
					return False

			return True

	def CanDoWork( self, bandwidth_tracker, expected_requests, expected_bytes, threshold = 30 ):
		with self._lock:
			for ( bandwidth_type, time_delta, max_allowed ) in self._rules:
				# Do not prohibit a raft of work starting or continuing because one small rule is over at this current second
				if time_delta is not None and time_delta <= threshold:
					continue

				# we don't want to do a tiny amount of work, we want to do a decent whack
				if bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS:
					max_allowed -= expected_requests

				elif bandwidth_type == HC.BANDWIDTH_TYPE_DATA:
					max_allowed -= expected_bytes

				if bandwidth_tracker.GetUsage( bandwidth_type, time_delta ) >= max_allowed:
					return False

			return True

	def CanStartRequest( self, bandwidth_tracker, threshold = 5 ):
		with self._lock:
			for ( bandwidth_type, time_delta, max_allowed ) in self._rules:
				# Do not prohibit a new job from starting just because the current download speed is 210/200KB/s
				ignore_rule = bandwidth_type == HC.BANDWIDTH_TYPE_DATA and time_delta is not None and time_delta <= threshold

				if ignore_rule:
					continue

				if bandwidth_tracker.GetUsage( bandwidth_type, time_delta ) >= max_allowed:
					return False

			return True

	def GetWaitingEstimate( self, bandwidth_tracker ):
		with self._lock:
			estimates = []

			for ( bandwidth_type, time_delta, max_allowed ) in self._rules:
				if bandwidth_tracker.GetUsage( bandwidth_type, time_delta ) >= max_allowed:
					estimates.append( bandwidth_tracker.GetWaitingEstimate( bandwidth_type, time_delta, max_allowed ) )

			if len( estimates ) == 0:
				return 0

			else:
				return max( estimates )

	def GetBandwidthStringsAndGaugeTuples( self, bandwidth_tracker, threshold = 600 ):
		with self._lock:
			rows = []

			rules_sorted = list( self._rules )

			def key( rule_tuple ):
				( bandwidth_type, time_delta, max_allowed ) = rule_tuple

				if time_delta is None:
					return -1

				else:
					return time_delta

			rules_sorted.sort( key = key )

			for ( bandwidth_type, time_delta, max_allowed ) in rules_sorted:
				time_is_less_than_threshold = time_delta is not None and time_delta <= threshold

				if time_is_less_than_threshold or max_allowed == 0:
					continue

				usage = bandwidth_tracker.GetUsage( bandwidth_type, time_delta )

				s = 'used '

				if bandwidth_type == HC.BANDWIDTH_TYPE_DATA:
					s += ConvertValueRangeToBytes( usage, max_allowed )

				elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS:
					s += ConvertValueRangeToPrettyString( usage, max_allowed ) + ' requests'

				if time_delta is None:
					s += ' this month'

				else:
					s += ' in the past ' + TimeDeltaToPrettyTimeDelta( time_delta )

				rows.append( ( s, ( usage, max_allowed ) ) )

			return rows

	def GetRules( self ):
		with self._lock:
			return list( self._rules )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["BANDWIDTH_RULES"] ] = BandwidthRules

class BandwidthTracker( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["BANDWIDTH_TRACKER"]
	SERIALISABLE_NAME = 'Bandwidth Tracker'
	SERIALISABLE_VERSION = 1

	# I want to track and query using smaller periods even when the total time delta is larger than the next step up to increase granularity
	# for instance, querying minutes for 90 mins time delta is more smooth than watching a juddery sliding two hour window
	MAX_SECONDS_TIME_DELTA = 240
	MAX_MINUTES_TIME_DELTA = 180 * 60
	MAX_HOURS_TIME_DELTA = 72 * 3600
	MAX_DAYS_TIME_DELTA = 31 * 86400

	CACHE_MAINTENANCE_TIME_DELTA = 120

	MIN_TIME_DELTA_FOR_USER = 10

	def __init__( self ):
		SerialisableBase.__init__( self )

		self._lock = threading.Lock()

		self._next_cache_maintenance_timestamp = GetNow() + self.CACHE_MAINTENANCE_TIME_DELTA

		self._months_bytes = collections.Counter()
		self._days_bytes = collections.Counter()
		self._hours_bytes = collections.Counter()
		self._minutes_bytes = collections.Counter()
		self._seconds_bytes = collections.Counter()

		self._months_requests = collections.Counter()
		self._days_requests = collections.Counter()
		self._hours_requests = collections.Counter()
		self._minutes_requests = collections.Counter()
		self._seconds_requests = collections.Counter()

	def _GetSerialisableInfo( self ):
		dicts_flat = []

		for d in ( self._months_bytes, self._days_bytes, self._hours_bytes, self._minutes_bytes, self._seconds_bytes, self._months_requests, self._days_requests, self._hours_requests, self._minutes_requests, self._seconds_requests ):
			dicts_flat.append( list( d.items() ) )

		return dicts_flat

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		counters = [ collections.Counter( dict( flat_dict ) ) for flat_dict in serialisable_info ]

		# unusual error someone reported by email--it came back an empty list, fugg
		if len( counters ) != 10:
			return

		self._months_bytes = counters[ 0 ]
		self._days_bytes = counters[ 1 ]
		self._hours_bytes = counters[ 2 ]
		self._minutes_bytes = counters[ 3 ]
		self._seconds_bytes = counters[ 4 ]

		self._months_requests = counters[ 5 ]
		self._days_requests = counters[ 6 ]
		self._hours_requests = counters[ 7 ]
		self._minutes_requests = counters[ 8 ]
		self._seconds_requests = counters[ 9 ]

	def _GetCurrentDateTime( self ):
		# keep getnow in here for the moment to aid in testing, which patches it to do time shifting
		return datetime.datetime.utcfromtimestamp( GetNow() )

	def _GetWindowAndCounter( self, bandwidth_type, time_delta ):
		if bandwidth_type == HC.BANDWIDTH_TYPE_DATA:
			if time_delta < self.MAX_SECONDS_TIME_DELTA:
				window = 0
				counter = self._seconds_bytes

			elif time_delta < self.MAX_MINUTES_TIME_DELTA:
				window = 60
				counter = self._minutes_bytes

			elif time_delta < self.MAX_HOURS_TIME_DELTA:
				window = 3600
				counter = self._hours_bytes

			else:
				window = 86400
				counter = self._days_bytes

		elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS:
			if time_delta < self.MAX_SECONDS_TIME_DELTA:
				window = 0
				counter = self._seconds_requests

			elif time_delta < self.MAX_MINUTES_TIME_DELTA:
				window = 60
				counter = self._minutes_requests

			elif time_delta < self.MAX_HOURS_TIME_DELTA:
				window = 3600
				counter = self._hours_requests

			else:
				window = 86400
				counter = self._days_requests

		return ( window, counter )

	def _GetMonthTime( self, dt ):
		( year, month ) = ( dt.year, dt.month )

		month_dt = datetime.datetime( year, month, 1 )

		month_time = int( calendar.timegm( month_dt.timetuple() ) )

		return month_time

	def _GetRawUsage( self, bandwidth_type, time_delta ):
		if time_delta is None:
			dt = self._GetCurrentDateTime()

			month_time = self._GetMonthTime( dt )

			if bandwidth_type == HC.BANDWIDTH_TYPE_DATA:
				return self._months_bytes[ month_time ]

			elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS:
				return self._months_requests[ month_time ]

		( window, counter ) = self._GetWindowAndCounter( bandwidth_type, time_delta )

		if time_delta == 1:
			# the case of 1 poses a problem as our min block width is also 1. we can't have a window of 0.1s to make the transition smooth
			# if we include the last second's data in an effort to span the whole previous 1000ms, we end up not doing anything until the next second rolls over
			# this causes 50% consumption as we consume in the second after the one we verified was clear
			# so, let's just check the current second and be happy with it

			now = GetNow()

			if now in counter:
				return counter[ now ]

			else:
				return 0

		else:
			# we need the 'window' because this tracks brackets from the first timestamp and we want to include if 'since' lands anywhere in the bracket
			# e.g. if it is 1200 and we want the past 1,000, we also need the bracket starting at 0, which will include 200-999

			search_time_delta = time_delta + window

			since = GetNow() - search_time_delta

			return sum( ( value for ( timestamp, value ) in list( counter.items() ) if timestamp >= since ) )

	def _GetTimes( self, dt ):
		# collapse each time portion to the latest timestamp it covers

		( year, month, day, hour, minute ) = ( dt.year, dt.month, dt.day, dt.hour, dt.minute )

		month_dt = datetime.datetime( year, month, 1 )
		day_dt = datetime.datetime( year, month, day )
		hour_dt = datetime.datetime( year, month, day, hour )
		minute_dt = datetime.datetime( year, month, day, hour, minute )

		month_time = int( calendar.timegm( month_dt.timetuple() ) )
		day_time = int( calendar.timegm( day_dt.timetuple() ) )
		hour_time = int( calendar.timegm( hour_dt.timetuple() ) )
		minute_time = int( calendar.timegm( minute_dt.timetuple() ) )

		second_time = int( calendar.timegm( dt.timetuple() ) )

		return ( month_time, day_time, hour_time, minute_time, second_time )

	def _GetUsage( self, bandwidth_type, time_delta, for_user ):
		if for_user and time_delta is not None and bandwidth_type == HC.BANDWIDTH_TYPE_DATA and time_delta <= self.MIN_TIME_DELTA_FOR_USER:
			usage = self._GetWeightedApproximateUsage( time_delta )

		else:
			usage = self._GetRawUsage( bandwidth_type, time_delta )

		self._MaintainCache()

		return usage

	def _GetWeightedApproximateUsage( self, time_delta ):
		SEARCH_DELTA = self.MIN_TIME_DELTA_FOR_USER

		counter = self._seconds_bytes

		now = GetNow()

		since = now - SEARCH_DELTA

		valid_keys = [ key for key in list( counter.keys() ) if key >= since ]

		if len( valid_keys ) == 0:
			return 0

		# If we want the average speed over past five secs but nothing has happened in sec 4 and 5, we don't want to count them
		# otherwise your 1MB/s counts as 200KB/s

		earliest_timestamp = min( valid_keys )

		SAMPLE_DELTA = max( now - earliest_timestamp, 1 )

		total_bytes = sum( ( counter[ key ] for key in valid_keys ) )

		time_delta_average_per_sec = total_bytes / SAMPLE_DELTA

		return time_delta_average_per_sec * time_delta

	def _MaintainCache( self ):
		if TimeHasPassed( self._next_cache_maintenance_timestamp ):
			now = GetNow()

			oldest_second = now - self.MAX_SECONDS_TIME_DELTA
			oldest_minute = now - self.MAX_MINUTES_TIME_DELTA
			oldest_hour = now - self.MAX_HOURS_TIME_DELTA
			oldest_day = now - self.MAX_DAYS_TIME_DELTA

			def clear_counter( counter, timestamp ):
				bad_keys = [ key for key in list( counter.keys() ) if key < timestamp ]

				for bad_key in bad_keys:
					del counter[ bad_key ]

			clear_counter( self._days_bytes, oldest_day )
			clear_counter( self._days_requests, oldest_day )
			clear_counter( self._hours_bytes, oldest_hour )
			clear_counter( self._hours_requests, oldest_hour )
			clear_counter( self._minutes_bytes, oldest_minute )
			clear_counter( self._minutes_requests, oldest_minute )
			clear_counter( self._seconds_bytes, oldest_second )
			clear_counter( self._seconds_requests, oldest_second )

			self._next_cache_maintenance_timestamp = GetNow() + self.CACHE_MAINTENANCE_TIME_DELTA

	def GetCurrentMonthSummary( self ):
		with self._lock:
			num_bytes = self._GetUsage( HC.BANDWIDTH_TYPE_DATA, None, True )
			num_requests = self._GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, None, True )

			return 'used ' + ToHumanBytes( num_bytes ) + ' in ' + ToHumanInt( num_requests ) + ' requests this month'

	def GetMonthlyDataUsage( self ):
		with self._lock:
			result = []

			for ( month_time, usage ) in list( self._months_bytes.items() ):
				month_dt = datetime.datetime.utcfromtimestamp( month_time )

				# this generates zero-padded month, to keep this lexicographically sortable at the gui level
				date_str = month_dt.strftime( '%Y-%m' )

				result.append( ( date_str, usage ) )

			result.sort()

			return result

	def GetUsage( self, bandwidth_type, time_delta, for_user = False ):
		with self._lock:
			if time_delta == 0:
				return 0

			return self._GetUsage( bandwidth_type, time_delta, for_user )

	def GetWaitingEstimate( self, bandwidth_type, time_delta, max_allowed ):
		with self._lock:
			if time_delta is None: # this is monthly

				dt = self._GetCurrentDateTime()

				( year, month ) = ( dt.year, dt.month )

				next_month_year = year

				if month == 12:
					next_month_year += 1

				next_month = ( month % 12 ) + 1

				next_month_dt = datetime.datetime( next_month_year, next_month, 1 )

				next_month_time = int( calendar.timegm( next_month_dt.timetuple() ) )

				return GetTimeDeltaUntilTime( next_month_time )

			else:
				# we want the highest time_delta at which usage is >= than max_allowed
				# time_delta subtract that amount is the time we have to wait for usage to be less than max_allowed
				# e.g. if in the past 24 hours there was a bunch of usage 16 hours ago clogging it up, we'll have to wait ~8 hours

				( window, counter ) = self._GetWindowAndCounter( bandwidth_type, time_delta )

				time_delta_in_which_bandwidth_counts = time_delta + window

				time_and_values = list( counter.items() )

				time_and_values.sort( reverse = True )

				now = GetNow()
				usage = 0

				for ( timestamp, value ) in time_and_values:
					current_search_time_delta = now - timestamp

					if current_search_time_delta > time_delta_in_which_bandwidth_counts: # we are searching beyond our time delta. no need to wait

						break

					usage += value

					if usage >= max_allowed:
						return time_delta_in_which_bandwidth_counts - current_search_time_delta

				return 0

	def ReportDataUsed( self, num_bytes ):
		with self._lock:
			dt = self._GetCurrentDateTime()

			( month_time, day_time, hour_time, minute_time, second_time ) = self._GetTimes( dt )

			self._months_bytes[ month_time ] += num_bytes

			self._days_bytes[ day_time ] += num_bytes

			self._hours_bytes[ hour_time ] += num_bytes

			self._minutes_bytes[ minute_time ] += num_bytes

			self._seconds_bytes[ second_time ] += num_bytes

			self._MaintainCache()

	def ReportRequestUsed( self ):
		with self._lock:
			dt = self._GetCurrentDateTime()

			( month_time, day_time, hour_time, minute_time, second_time ) = self._GetTimes( dt )

			self._months_requests[ month_time ] += 1

			self._days_requests[ day_time ] += 1

			self._hours_requests[ hour_time ] += 1

			self._minutes_requests[ minute_time ] += 1

			self._seconds_requests[ second_time ] += 1

			self._MaintainCache()

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["BANDWIDTH_TRACKER"] ] = BandwidthTracker

def GenerateDefaultServiceDictionary( service_type ):
	dictionary = SerialisableDictionary()

	dictionary[ 'upnp_port' ] = None
	dictionary[ 'bandwidth_tracker' ] = BandwidthTracker()

	if service_type in HC.RESTRICTED_SERVICES:
		dictionary[ 'bandwidth_rules' ] = BandwidthRules()

		if service_type in HC.REPOSITORIES:
			metadata = Metadata()

			now = GetNow()

			update_hashes = []
			begin = 0
			end = now
			next_update_due = now + HC.UPDATE_DURATION

			metadata.AppendUpdate( update_hashes, begin, end, next_update_due )

			dictionary[ 'metadata' ] = metadata

			if service_type == HC.FILE_REPOSITORY:
				dictionary[ 'log_uploader_ips' ] = False
				dictionary[ 'max_storage' ] = None

		if service_type == HC.SERVER_ADMIN:
			dictionary[ 'server_bandwidth_tracker' ] = BandwidthTracker()
			dictionary[ 'server_bandwidth_rules' ] = BandwidthRules()

	return dictionary

def GenerateService( service_key, service_type, name, port, dictionary = None ):
	if dictionary is None:
		dictionary = GenerateDefaultServiceDictionary( service_type )

	if service_type == HC.SERVER_ADMIN:
		cl = ServerServiceAdmin

	elif service_type == HC.TAG_REPOSITORY:
		cl = ServerServiceRepositoryTag

	elif service_type == HC.FILE_REPOSITORY:
		cl = ServerServiceRepositoryFile

	return cl( service_key, service_type, name, port, dictionary )

def GenerateServiceFromSerialisableTuple( serialisable_info ):
	( service_key_encoded, service_type, name, port, dictionary_string ) = serialisable_info

	try:
		service_key = bytes.fromhex( service_key_encoded )

	except TypeError:
		raise InsufficientCredentialsException( 'Could not decode that service key!' )

	dictionary = CreateFromString( dictionary_string )

	return GenerateService( service_key, service_type, name, port, dictionary )

def GetPossiblePermissions( service_type ):
	permissions = []

	permissions.append( ( HC.CONTENT_TYPE_ACCOUNTS, [ None, HC.PERMISSION_ACTION_CREATE, HC.PERMISSION_ACTION_OVERRULE ] ) )
	permissions.append( ( HC.CONTENT_TYPE_ACCOUNT_TYPES, [ None, HC.PERMISSION_ACTION_OVERRULE ] ) )

	if service_type == HC.FILE_REPOSITORY:
		permissions.append( ( HC.CONTENT_TYPE_FILES, [ None, HC.PERMISSION_ACTION_PETITION, HC.PERMISSION_ACTION_CREATE, HC.PERMISSION_ACTION_OVERRULE ] ) )

	elif service_type == HC.TAG_REPOSITORY:
		permissions.append( ( HC.CONTENT_TYPE_MAPPINGS, [ None, HC.PERMISSION_ACTION_PETITION, HC.PERMISSION_ACTION_CREATE, HC.PERMISSION_ACTION_OVERRULE ] ) )
		permissions.append( ( HC.CONTENT_TYPE_TAG_PARENTS, [ None, HC.PERMISSION_ACTION_PETITION, HC.PERMISSION_ACTION_OVERRULE ] ) )
		permissions.append( ( HC.CONTENT_TYPE_TAG_SIBLINGS, [ None, HC.PERMISSION_ACTION_PETITION, HC.PERMISSION_ACTION_OVERRULE ] ) )

	elif service_type == HC.SERVER_ADMIN:
		permissions.append( ( HC.CONTENT_TYPE_SERVICES, [ None, HC.PERMISSION_ACTION_OVERRULE ] ) )

	return permissions

def DumpHydrusArgsToNetworkBytes( args ):
	if not isinstance( args, SerialisableBase ):
		args = SerialisableDictionary( args )

	if 'access_key' in args:
		args[ 'access_key' ] = args[ 'access_key' ].hex()

	if 'account' in args:
		args[ 'account' ] = Account.GenerateSerialisableTupleFromAccount( args[ 'account' ] )

	if 'accounts' in args:
		args[ 'accounts' ] = list( map( Account.GenerateSerialisableTupleFromAccount, args[ 'accounts' ] ) )

	if 'account_types' in args:
		args[ 'account_types' ] = [ account_type.ToSerialisableTuple() for account_type in args[ 'account_types' ] ]

	if 'registration_keys' in args:
		args[ 'registration_keys' ] = [ registration_key.hex() for registration_key in args[ 'registration_keys' ] ]

	if 'service_keys_to_access_keys' in args:
		args[ 'service_keys_to_access_keys' ] = [ ( service_key.hex(), access_key.hex() ) for ( service_key, access_key ) in list(args[ 'service_keys_to_access_keys' ].items() ) ]

	if 'services' in args:
		args[ 'services' ] = [ service.ToSerialisableTuple() for service in args[ 'services' ] ]

	network_bytes = args.DumpToNetworkBytes()

	return network_bytes

def DumpToGETQuery( args ):
	args = dict( args )

	if 'subject_identifier' in args:
		subject_identifier = args[ 'subject_identifier' ]

		del args[ 'subject_identifier' ]

		if subject_identifier.HasAccountKey():
			account_key = subject_identifier.GetData()

			args[ 'subject_account_key' ] = account_key

		elif subject_identifier.HasContent():
			content = subject_identifier.GetData()

			content_type = content.GetContentType()
			content_data = content.GetContentData()

			if content_type == HC.CONTENT_TYPE_FILES:
				hash = content_data[0]

				args[ 'subject_hash' ] = hash

			elif content_type == HC.CONTENT_TYPE_MAPPING:
				( tag, hash ) = content_data

				args[ 'subject_hash' ] = hash
				args[ 'subject_tag' ] = tag

	for name in INT_PARAMS:
		if name in args:
			args[ name ] = str( args[ name ] )

	for name in BYTE_PARAMS:
		if name in args:
			args[ name ] = args[ name ].hex()

	for name in STRING_PARAMS:
		if name in args:
			args[ name ] = urllib.parse.quote( args[ name ] )

	query = '&'.join( [ key + '=' + value for ( key, value ) in args.items() ] )

	return query

def ParseNetworkBytesToParsedHydrusArgs( network_bytes ):
	if len( network_bytes ) == 0:
		return SerialisableDictionary()

	args = CreateFromNetworkBytes( network_bytes )

	if not isinstance( args, dict ):
		raise BadRequestException( 'The given parameter did not seem to be a JSON Object!' )

	args = ParsedRequestArguments( args )

	if 'access_key' in args:
		args[ 'access_key' ] = bytes.fromhex( args[ 'access_key' ] )

	if 'account' in args:
		args[ 'account' ] = Account.GenerateAccountFromSerialisableTuple( args[ 'account' ] )

	if 'accounts' in args:
		account_tuples = args[ 'accounts' ]

		args[ 'accounts' ] = list( map( Account.GenerateAccountFromSerialisableTuple, account_tuples ) )

	if 'account_types' in args:
		account_type_tuples = args[ 'account_types' ]

		args[ 'account_types' ] = list( map( AccountType.GenerateAccountTypeFromSerialisableTuple, account_type_tuples ) )

	if 'registration_keys' in args:
		args[ 'registration_keys' ] = [ bytes.fromhex( encoded_registration_key ) for encoded_registration_key in args[ 'registration_keys' ] ]

	if 'service_keys_to_access_keys' in args:
		args[ 'service_keys_to_access_keys' ] = { bytes.fromhex( encoded_service_key ) : bytes.fromhex( encoded_access_key ) for ( encoded_service_key, encoded_access_key ) in args[ 'service_keys_to_access_keys' ] }

	if 'services' in args:
		service_tuples = args[ 'services' ]

		args[ 'services' ] = list( map( GenerateServiceFromSerialisableTuple, service_tuples ) )

	return args

def ParseHydrusNetworkGETArgs( requests_args ):
	args = ParseTwistedRequestGETArgs( requests_args, INT_PARAMS, BYTE_PARAMS, STRING_PARAMS, JSON_PARAMS )

	if 'subject_account_key' in args:
		args[ 'subject_identifier' ] = AccountIdentifier( account_key = args[ 'subject_account_key' ] )

	elif 'subject_hash' in args:
		hash = args[ 'subject_hash' ]

		if 'subject_tag' in args:
			tag = args[ 'subject_tag' ]

			content = Content( HC.CONTENT_TYPE_MAPPING, ( tag, hash ) )

		else:
			content = Content( HC.CONTENT_TYPE_FILES, [ hash ] )

		args[ 'subject_identifier' ] = AccountIdentifier( content = content )

	return args

class Account( object ):
	def __init__( self, account_key, account_type, created, expires, banned_info = None, bandwidth_tracker = None ):

		if banned_info is None:
			banned_info = None # stupid, but keep it in case we change this

		if bandwidth_tracker is None:
			bandwidth_tracker = BandwidthTracker()

		SerialisableBase.__init__( self )

		self._lock = threading.Lock()

		self._account_key = account_key
		self._account_type = account_type
		self._created = created
		self._expires = expires
		self._banned_info = banned_info
		self._bandwidth_tracker = bandwidth_tracker

		self._dirty = False

	def __repr__( self ):
		return 'Account: ' + self._account_type.GetTitle()

	def __str__( self ):
		return self.__repr__()

	def _CheckFunctional( self ):
		if self._created == 0:
			raise InsufficientCredentialsException( 'account is unsynced' )

		if self._account_type.HasPermission( HC.CONTENT_TYPE_SERVICES, HC.PERMISSION_ACTION_OVERRULE ):
			return # admins can do anything

		if self._IsBanned():
			raise InsufficientCredentialsException( 'This account is banned: ' + self._GetBannedString() )

		if self._IsExpired():
			raise InsufficientCredentialsException( 'This account is expired: ' + self._GetExpiresString() )

		if not self._account_type.BandwidthOK( self._bandwidth_tracker ):
			raise InsufficientCredentialsException( 'account has exceeded bandwidth' )

	def _GetBannedString( self ):
		if self._banned_info is None:
			return 'not banned'

		else:
			( reason, created, expires ) = self._banned_info

			return 'banned ' + TimestampToPrettyTimeDelta( created ) + ', ' + ConvertTimestampToPrettyExpires( expires ) + ' because: ' + reason

	def _GetExpiresString( self ):
		return ConvertTimestampToPrettyExpires( self._expires )

	def _IsBanned( self ):
		if self._banned_info is None:
			return False

		else:
			( reason, created, expires ) = self._banned_info

			if expires is None:
				return True

			else:
				if TimeHasPassed( expires ):
					self._banned_info = None

				else:
					return True

	def _IsExpired( self ):
		if self._expires is None:
			return False

		else:
			return TimeHasPassed( self._expires )

	def _SetDirty( self ):
		self._dirty = True

	def Ban( self, reason, created, expires ):
		with self._lock:
			self._banned_info = ( reason, created, expires )

			self._SetDirty()

	def CheckFunctional( self ):
		with self._lock:
			self._CheckFunctional()

	def CheckPermission( self, content_type, action ):
		with self._lock:
			if not self._account_type.HasPermission( content_type, action ):
				raise InsufficientCredentialsException( 'You do not have permission to do that.' )

	def GetAccountKey( self ):
		with self._lock:
			return self._account_key

	def GetAccountType( self ):
		with self._lock:
			return self._account_type

	def GetBandwidthCurrentMonthSummary( self ):
		with self._lock:
			return self._bandwidth_tracker.GetCurrentMonthSummary()

	def GetBandwidthStringsAndGaugeTuples( self ):
		with self._lock:
			return self._account_type.GetBandwidthStringsAndGaugeTuples( self._bandwidth_tracker )

	def GetExpiresString( self ):
		with self._lock:
			if self._IsBanned():
				return self._GetBannedString()

			else:
				return self._GetExpiresString()

	def GetStatusString( self ):
		with self._lock:
			try:
				self._CheckFunctional()

				return 'account is functional'

			except Exception as e:
				return str( e )

	def HasPermission( self, content_type, action ):
		with self._lock:
			return self._account_type.HasPermission( content_type, action )

	def IsDirty( self ):
		with self._lock:
			return self._dirty

	def IsFunctional( self ):
		with self._lock:
			try:
				self._CheckFunctional()

				return True

			except Exception:
				return False

	def ReportDataUsed( self, num_bytes ):
		with self._lock:
			self._bandwidth_tracker.ReportDataUsed( num_bytes )

			self._SetDirty()

	def ReportRequestUsed( self ):
		with self._lock:
			self._bandwidth_tracker.ReportRequestUsed()

			self._SetDirty()

	def SetClean( self ):
		with self._lock:
			self._dirty = False

	def SetExpires( self, expires ):
		with self._lock:
			self._expires = expires

			self._SetDirty()

	def ToString( self ):
		with self._lock:
			return self._account_type.GetTitle() + ' -- created ' + TimestampToPrettyTimeDelta( self._created )

	def ToTuple( self ):
		with self._lock:
			return ( self._account_key, self._account_type, self._created, self._expires, self._banned_info, self._bandwidth_tracker )

	def Unban( self ):
		with self._lock:
			self._banned_info = None

			self._SetDirty()

	@staticmethod

	def GenerateAccountFromSerialisableTuple( serialisable_info ):
		( account_key_encoded, account_type_serialisable_tuple, created, expires, dictionary_string ) = serialisable_info

		account_key = bytes.fromhex( account_key_encoded )
		account_type = AccountType.GenerateAccountTypeFromSerialisableTuple( account_type_serialisable_tuple )
		dictionary = CreateFromString( dictionary_string )

		return Account.GenerateAccountFromTuple( ( account_key, account_type, created, expires, dictionary ) )

	@staticmethod

	def GenerateAccountFromTuple( serialisable_info ):
		( account_key, account_type, created, expires, dictionary ) = serialisable_info

		banned_info = dictionary[ 'banned_info' ]
		bandwidth_tracker = dictionary[ 'bandwidth_tracker' ]

		return Account( account_key, account_type, created, expires, banned_info, bandwidth_tracker )

	@staticmethod

	def GenerateSerialisableTupleFromAccount( account ):
		( account_key, account_type, created, expires, dictionary ) = Account.GenerateTupleFromAccount( account )

		account_key_encoded = account_key.hex()

		serialisable_account_type = account_type.ToSerialisableTuple()

		dictionary_string = dictionary.DumpToString()

		return ( account_key_encoded, serialisable_account_type, created, expires, dictionary_string )

	@staticmethod

	def GenerateTupleFromAccount( account ):
		( account_key, account_type, created, expires, banned_info, bandwidth_tracker ) = account.ToTuple()

		dictionary = SerialisableDictionary()

		dictionary[ 'banned_info' ] = banned_info

		dictionary[ 'bandwidth_tracker' ] = bandwidth_tracker

		dictionary = dictionary.Duplicate()

		return ( account_key, account_type, created, expires, dictionary )

	@staticmethod

	def GenerateUnknownAccount( account_key = b'' ):
		account_type = AccountType.GenerateUnknownAccountType()
		created = 0
		expires = None

		unknown_account = Account( account_key, account_type, created, expires )

		return unknown_account

class AccountType( object ):
	def __init__( self, account_type_key, title, dictionary ):

		SerialisableBase.__init__( self )

		self._account_type_key = account_type_key
		self._title = title

		self._LoadFromDictionary( dictionary )

	def __repr__( self ):
		return 'AccountType: ' + self._title

	def __str__( self ):
		return self.__repr__()

	def _GetSerialisableDictionary( self ):
		dictionary = SerialisableDictionary()

		dictionary[ 'permissions' ] = list( self._permissions.items() )

		dictionary[ 'bandwidth_rules' ] = self._bandwidth_rules

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		self._permissions = dict( dictionary[ 'permissions' ] )

		self._bandwidth_rules = dictionary[ 'bandwidth_rules' ]

	def BandwidthOK( self, bandwidth_tracker ):
		return self._bandwidth_rules.CanStartRequest( bandwidth_tracker )

	def HasPermission( self, content_type, permission ):
		if content_type not in self._permissions:
			return False

		my_permission = self._permissions[ content_type ]

		if permission == HC.PERMISSION_ACTION_OVERRULE:
			return my_permission == HC.PERMISSION_ACTION_OVERRULE

		elif permission == HC.PERMISSION_ACTION_CREATE:
			return my_permission in ( HC.PERMISSION_ACTION_CREATE, HC.PERMISSION_ACTION_OVERRULE )

		elif permission == HC.PERMISSION_ACTION_PETITION:
			return my_permission in ( HC.PERMISSION_ACTION_PETITION, HC.PERMISSION_ACTION_CREATE, HC.PERMISSION_ACTION_OVERRULE )

		return False

	def GetBandwidthStringsAndGaugeTuples( self, bandwidth_tracker ):
		return self._bandwidth_rules.GetBandwidthStringsAndGaugeTuples( bandwidth_tracker )

	def GetAccountTypeKey( self ):
		return self._account_type_key

	def GetPermissionStrings( self ):
		s = []

		for ( content_type, action ) in list( self._permissions.items() ):
			s.append( HC.permission_pair_string_lookup[ ( content_type, action ) ] )

		return s

	def GetTitle( self ):
		return self._title

	def ToDictionaryTuple( self ):
		dictionary = self._GetSerialisableDictionary()

		return ( self._account_type_key, self._title, dictionary )

	def ToSerialisableTuple( self ):
		dictionary = self._GetSerialisableDictionary()

		dictionary_string = dictionary.DumpToString()

		return ( self._account_type_key.hex(), self._title, dictionary_string )

	def ToTuple( self ):
		return ( self._account_type_key, self._title, self._permissions, self._bandwidth_rules )

	@staticmethod

	def GenerateAccountTypeFromParameters( account_type_key, title, permissions, bandwidth_rules ):
		dictionary = SerialisableDictionary()

		dictionary[ 'permissions' ] = list( permissions.items() )
		dictionary[ 'bandwidth_rules' ] = bandwidth_rules

		return AccountType( account_type_key, title, dictionary )

	@staticmethod

	def GenerateAccountTypeFromSerialisableTuple( serialisable_info ):
		( account_type_key_encoded, title, dictionary_string ) = serialisable_info

		try:
			account_type_key = bytes.fromhex( account_type_key_encoded )

		except TypeError:
			raise InsufficientCredentialsException( 'Could not decode that account type key!' )

		dictionary = CreateFromString( dictionary_string )

		return AccountType( account_type_key, title, dictionary )

	@staticmethod

	def GenerateAdminAccountType( service_type ):
		bandwidth_rules = BandwidthRules()

		permissions = {}

		permissions[ HC.CONTENT_TYPE_ACCOUNTS ] = HC.PERMISSION_ACTION_OVERRULE
		permissions[ HC.CONTENT_TYPE_ACCOUNT_TYPES ] = HC.PERMISSION_ACTION_OVERRULE

		if service_type in HC.REPOSITORIES:
			for content_type in HC.REPOSITORY_CONTENT_TYPES:
				permissions[ content_type ] = HC.PERMISSION_ACTION_OVERRULE

		elif service_type == HC.SERVER_ADMIN:
			permissions[ HC.CONTENT_TYPE_SERVICES ] = HC.PERMISSION_ACTION_OVERRULE

		else:
			raise NotImplementedError( 'Do not have a default admin account type set up for this service yet!' )

		account_type = AccountType.GenerateNewAccountTypeFromParameters( 'administrator', permissions, bandwidth_rules )

		return account_type

	@staticmethod

	def GenerateNewAccountTypeFromParameters( title, permissions, bandwidth_rules ):
		account_type_key = GenerateKey()

		return AccountType.GenerateAccountTypeFromParameters( account_type_key, title, permissions, bandwidth_rules )

	@staticmethod

	def GenerateUnknownAccountType():
		title = 'unknown account'
		permissions = {}

		bandwidth_rules = BandwidthRules()
		bandwidth_rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, None, 0 )

		unknown_account_type = AccountType.GenerateNewAccountTypeFromParameters( title, permissions, bandwidth_rules )

		return unknown_account_type

class ClientToServerUpdate( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["CLIENT_TO_SERVER_UPDATE"]
	SERIALISABLE_NAME = 'Client To Server Update'
	SERIALISABLE_VERSION = 1

	def __init__( self ):
		SerialisableBase.__init__( self )

		self._actions_to_contents_and_reasons = collections.defaultdict( list )

	def _GetSerialisableInfo( self ):
		serialisable_info = []

		for ( action, contents_and_reasons ) in list( self._actions_to_contents_and_reasons.items() ):
			serialisable_contents_and_reasons = [ ( content.GetSerialisableTuple(), reason ) for ( content, reason ) in contents_and_reasons ]

			serialisable_info.append( ( action, serialisable_contents_and_reasons ) )

		return serialisable_info

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		for ( action, serialisable_contents_and_reasons ) in serialisable_info:
			contents_and_reasons = [ ( CreateFromSerialisableTuple( serialisable_content ), reason ) for ( serialisable_content, reason ) in serialisable_contents_and_reasons ]

			self._actions_to_contents_and_reasons[ action ] = contents_and_reasons

	def AddContent( self, action, content, reason = None ):
		if reason is None:
			reason = ''

		self._actions_to_contents_and_reasons[ action ].append( ( content, reason ) )

	def GetClientsideContentUpdates( self ):
		content_updates = []

		for ( action, clientside_action ) in ( ( HC.CONTENT_UPDATE_PEND, HC.CONTENT_UPDATE_ADD ), ( HC.CONTENT_UPDATE_PETITION, HC.CONTENT_UPDATE_DELETE ) ):
			for ( content, reason ) in self._actions_to_contents_and_reasons[ action ]:
				content_type = content.GetContentType()
				content_data = content.GetContentData()

				content_update = ContentUpdate( content_type, clientside_action, content_data )

				content_updates.append( content_update )

		return content_updates

	def GetContentDataIterator( self, content_type, action ):
		contents_and_reasons = self._actions_to_contents_and_reasons[ action ]

		for ( content, reason ) in contents_and_reasons:
			if content.GetContentType() == content_type:
				yield ( content.GetContentData(), reason )

	def GetHashes( self ):
		hashes = set()

		for contents_and_reasons in list( self._actions_to_contents_and_reasons.values() ):
			for ( content, reason ) in contents_and_reasons:
				hashes.update( content.GetHashes() )

		return hashes

	def HasContent( self ):
		return len( self._actions_to_contents_and_reasons ) > 0

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["CLIENT_TO_SERVER_UPDATE"] ] = ClientToServerUpdate

class Content( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["CONTENT"]
	SERIALISABLE_NAME = 'Content'
	SERIALISABLE_VERSION = 1

	def __init__( self, content_type = None, content_data = None ):
		SerialisableBase.__init__( self )

		self._content_type = content_type
		self._content_data = content_data

	def __eq__( self, other ): return self.__hash__() == other.__hash__()

	def __hash__( self ): return ( self._content_type, self._content_data ).__hash__()

	def __ne__( self, other ): return self.__hash__() != other.__hash__()

	def __repr__( self ): return 'Content: ' + self.ToString()

	def _GetSerialisableInfo( self ):
		def EncodeHashes( hs ):
			return [ h.hex() for h in hs ]

		if self._content_type == HC.CONTENT_TYPE_FILES:
			hashes = self._content_data

			serialisable_content = EncodeHashes( hashes )

		elif self._content_type == HC.CONTENT_TYPE_MAPPING:
			( tag, hash ) = self._content_data

			serialisable_content = ( tag, hash.hex() )

		elif self._content_type == HC.CONTENT_TYPE_MAPPINGS:
			( tag, hashes ) = self._content_data

			serialisable_content = ( tag, EncodeHashes( hashes ) )

		elif self._content_type in ( HC.CONTENT_TYPE_TAG_PARENTS, HC.CONTENT_TYPE_TAG_SIBLINGS ):
			( old_tag, new_tag ) = self._content_data

			serialisable_content = ( old_tag, new_tag )

		return ( self._content_type, serialisable_content )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		def DecodeHashes( hs ):
			return [ bytes.fromhex( h ) for h in hs ]

		( self._content_type, serialisable_content ) = serialisable_info

		if self._content_type == HC.CONTENT_TYPE_FILES:
			serialisable_hashes = serialisable_content

			self._content_data = DecodeHashes( serialisable_hashes )

		elif self._content_type == HC.CONTENT_TYPE_MAPPING:
			( tag, serialisable_hash ) = serialisable_content

			self._content_data = ( tag, bytes.fromhex( serialisable_hash ) )

		elif self._content_type == HC.CONTENT_TYPE_MAPPINGS:
			( tag, serialisable_hashes ) = serialisable_content

			self._content_data = ( tag, DecodeHashes( serialisable_hashes ) )

		elif self._content_type in ( HC.CONTENT_TYPE_TAG_PARENTS, HC.CONTENT_TYPE_TAG_SIBLINGS ):
			( old_tag, new_tag ) = serialisable_content

			self._content_data = ( old_tag, new_tag )

	def GetContentData( self ):
		return self._content_data

	def GetContentType( self ):
		return self._content_type

	def GetHashes( self ):
		if self._content_type == HC.CONTENT_TYPE_FILES:
			hashes = self._content_data

		elif self._content_type == HC.CONTENT_TYPE_MAPPING:
			( tag, hash ) = self._content_data

			return [ hash ]

		elif self._content_type == HC.CONTENT_TYPE_MAPPINGS:
			( tag, hashes ) = self._content_data

		else:
			hashes = []

		return hashes

	def GetVirtualWeight( self ):
		if self._content_type in ( HC.CONTENT_TYPE_FILES, HC.CONTENT_TYPE_MAPPINGS ):
			return len( self.GetHashes() )

		elif self._content_type == HC.CONTENT_TYPE_TAG_PARENTS:
			return 5000

		elif self._content_type == HC.CONTENT_TYPE_TAG_SIBLINGS:
			return 5

		elif self._content_type == HC.CONTENT_TYPE_MAPPING:
			return 1

	def HasHashes( self ):
		return self._content_type in ( HC.CONTENT_TYPE_FILES, HC.CONTENT_TYPE_MAPPING, HC.CONTENT_TYPE_MAPPINGS )

	def ToString( self ):
		if self._content_type == HC.CONTENT_TYPE_FILES:
			hashes = self._content_data

			text = 'FILES: ' + ToHumanInt( len( hashes ) ) + ' files'

		elif self._content_type == HC.CONTENT_TYPE_MAPPING:
			( tag, hash ) = self._content_data

			text = 'MAPPING: ' + tag + ' for ' + hash.hex()

		elif self._content_type == HC.CONTENT_TYPE_MAPPINGS:
			( tag, hashes ) = self._content_data

			text = 'MAPPINGS: ' + tag + ' for ' + ToHumanInt( len( hashes ) ) + ' files'

		elif self._content_type == HC.CONTENT_TYPE_TAG_PARENTS:
			( child, parent ) = self._content_data

			text = 'PARENT: ' '"' + child + '" -> "' + parent + '"'

		elif self._content_type == HC.CONTENT_TYPE_TAG_SIBLINGS:
			( old_tag, new_tag ) = self._content_data

			text = 'SIBLING: ' + '"' + old_tag + '" -> "' + new_tag + '"'

		return text

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["CONTENT"] ] = Content

class NetworkContentUpdate( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["CONTENT_UPDATE"]
	SERIALISABLE_NAME = 'Content Update'
	SERIALISABLE_VERSION = 1

	def __init__( self ):
		SerialisableBase.__init__( self )

		self._content_data = {}

	def _GetContent( self, content_type, action ):
		if content_type in self._content_data:
			if action in self._content_data[ content_type ]:
				return self._content_data[ content_type ][ action ]

		return []

	def _GetSerialisableInfo( self ):
		serialisable_info = []

		for ( content_type, actions_to_datas ) in list( self._content_data.items() ):
			serialisable_actions_to_datas = list( actions_to_datas.items() )

			serialisable_info.append( ( content_type, serialisable_actions_to_datas ) )

		return serialisable_info

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		for ( content_type, serialisable_actions_to_datas ) in serialisable_info:
			actions_to_datas = dict( serialisable_actions_to_datas )

			self._content_data[ content_type ] = actions_to_datas

	def AddRow( self, row ):
		( content_type, action, data ) = row

		if content_type not in self._content_data:
			self._content_data[ content_type ] = {}

		if action not in self._content_data[ content_type ]:
			self._content_data[ content_type ][ action ] = []

		self._content_data[ content_type ][ action ].append( data )

	def GetDeletedFiles( self ):
		return self._GetContent( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE )

	def GetDeletedMappings( self ):
		return self._GetContent( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_DELETE )

	def GetDeletedTagParents( self ):
		return self._GetContent( HC.CONTENT_TYPE_TAG_PARENTS, HC.CONTENT_UPDATE_DELETE )

	def GetDeletedTagSiblings( self ):
		return self._GetContent( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.CONTENT_UPDATE_DELETE )

	def GetNewFiles( self ):
		return self._GetContent( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ADD )

	def GetNewMappings( self ):
		return self._GetContent( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD )

	def GetNewTagParents( self ):
		return self._GetContent( HC.CONTENT_TYPE_TAG_PARENTS, HC.CONTENT_UPDATE_ADD )

	def GetNewTagSiblings( self ):
		return self._GetContent( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.CONTENT_UPDATE_ADD )

	def GetNumRows( self ):
		num = 0

		for content_type in self._content_data:
			for action in self._content_data[ content_type ]:
				data = self._content_data[ content_type ][ action ]

				if content_type == HC.CONTENT_TYPE_MAPPINGS:
					num_rows = sum( ( len( hash_ids ) for ( tag_id, hash_ids ) in data ) )

				else:
					num_rows = len( data )

				num += num_rows

		return num

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["CONTENT_UPDATE"] ] = ContentUpdate

class NetworkCredentials( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["CREDENTIALS"]
	SERIALISABLE_NAME = 'Credentials'
	SERIALISABLE_VERSION = 1

	def __init__( self, host = None, port = None, access_key = None ):
		SerialisableBase.__init__( self )

		self._host = host
		self._port = port
		self._access_key = access_key

	def __eq__( self, other ):
		return self.__hash__() == other.__hash__()

	def __hash__( self ):
		return ( self._host, self._port, self._access_key ).__hash__()

	def __ne__( self, other ):
		return self.__hash__() != other.__hash__()

	def __repr__( self ):
		return 'Credentials: ' + str( ( self._host, self._port, self._access_key.hex() ) )

	def _GetSerialisableInfo( self ):
		if self._access_key is None:
			serialisable_access_key = self._access_key

		else:
			serialisable_access_key = self._access_key.hex()

		return ( self._host, self._port, serialisable_access_key )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self._host, self._port, serialisable_access_key ) = serialisable_info

		if serialisable_access_key is None:
			self._access_key = serialisable_access_key

		else:
			self._access_key = bytes.fromhex( serialisable_access_key )

	def GetAccessKey( self ):
		return self._access_key

	def GetAddress( self ):
		return ( self._host, self._port )

	def GetConnectionString( self ):
		connection_string = ''

		if self.HasAccessKey():
			connection_string += self._access_key.hex() + '@'

		connection_string += self._host + ':' + str( self._port )

		return connection_string

	def HasAccessKey( self ):
		return self._access_key is not None

	def SetAccessKey( self, access_key ):
		if access_key == '':
			access_key = None

		self._access_key = access_key

	def SetAddress( self, host, port ):
		self._host = host
		self._port = port

	@staticmethod

	def GenerateCredentialsFromConnectionString( connection_string ):
		( host, port, access_key ) = NetworkCredentials.ParseConnectionString( connection_string )

		return NetworkCredentials( host, port, access_key )

	@staticmethod

	def ParseConnectionString( connection_string ):
		if connection_string is None:
			return ( 'hostname', 80, None )

		if '@' in connection_string:
			( access_key_encoded, address ) = connection_string.split( '@', 1 )

			try:
				access_key = bytes.fromhex( access_key_encoded )

			except TypeError:
				raise DataMissing( 'Could not parse that access key! It should be a 64 character hexadecimal string!' )

			if access_key == '':
				access_key = None

		else:
			access_key = None

		if ':' in connection_string:
			( host, port ) = connection_string.split( ':', 1 )

			try:
				port = int( port )

				if port < 0 or port > 65535:
					raise ValueError()

			except ValueError:
				raise DataMissing( 'Could not parse that port! It should be an integer between 0 and 65535!' )

		if host == 'localhost':
			host = '127.0.0.1'

		return ( host, port, access_key )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["CREDENTIALS"] ] = NetworkCredentials

class DefinitionsUpdate( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["DEFINITIONS_UPDATE"]
	SERIALISABLE_NAME = 'Definitions Update'
	SERIALISABLE_VERSION = 1

	def __init__( self ):
		SerialisableBase.__init__( self )

		self._hash_ids_to_hashes = {}
		self._tag_ids_to_tags = {}

	def _GetSerialisableInfo( self ):
		serialisable_info = []

		if len( self._hash_ids_to_hashes ) > 0:
			serialisable_info.append( ( HC.DEFINITIONS_TYPE_HASHES, [ ( hash_id, hash.hex() ) for ( hash_id, hash ) in list( self._hash_ids_to_hashes.items() ) ] ) )

		if len( self._tag_ids_to_tags ) > 0:
			serialisable_info.append( ( HC.DEFINITIONS_TYPE_TAGS, list( self._tag_ids_to_tags.items() ) ) )

		return serialisable_info

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		for ( definition_type, definitions ) in serialisable_info:
			if definition_type == HC.DEFINITIONS_TYPE_HASHES:
				self._hash_ids_to_hashes = { hash_id : bytes.fromhex( encoded_hash ) for ( hash_id, encoded_hash ) in definitions }

			elif definition_type == HC.DEFINITIONS_TYPE_TAGS:
				self._tag_ids_to_tags = { tag_id : tag for ( tag_id, tag ) in definitions }

	def AddRow( self, row ):
		( definitions_type, key, value ) = row

		if definitions_type == HC.DEFINITIONS_TYPE_HASHES:
			self._hash_ids_to_hashes[ key ] = value

		elif definitions_type == HC.DEFINITIONS_TYPE_TAGS:
			self._tag_ids_to_tags[ key ] = value

	def GetHashIdsToHashes( self ):
		return self._hash_ids_to_hashes

	def GetNumRows( self ):
		return len( self._hash_ids_to_hashes ) + len( self._tag_ids_to_tags )

	def GetTagIdsToTags( self ):
		return self._tag_ids_to_tags

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["DEFINITIONS_UPDATE"] ] = DefinitionsUpdate

class Metadata( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["METADATA"]
	SERIALISABLE_NAME = 'Metadata'
	SERIALISABLE_VERSION = 1

	CLIENT_DELAY = 20 * 60

	def __init__( self, metadata = None, next_update_due = None ):
		if metadata is None:
			metadata = {}

		if next_update_due is None:
			next_update_due = 0

		SerialisableBase.__init__( self )

		self._lock = threading.Lock()

		self._metadata = metadata
		self._next_update_due = next_update_due

		self._update_hashes = set()

	def _GetNextUpdateDueTime( self, from_client = False ):
		delay = 0

		if from_client:
			delay = self.CLIENT_DELAY

		return self._next_update_due + delay

	def _GetSerialisableInfo( self ):
		serialisable_metadata = [ ( update_index, [ update_hash.hex() for update_hash in update_hashes ], begin, end ) for ( update_index, ( update_hashes, begin, end ) ) in list( self._metadata.items() ) ]

		return ( serialisable_metadata, self._next_update_due )

	def _GetUpdateHashes( self, update_index ):
		( update_hashes, begin, end ) = self._GetUpdate( update_index )

		return update_hashes

	def _GetUpdate( self, update_index ):
		if update_index not in self._metadata:
			raise DataMissing( 'That update does not exist!' )

		return self._metadata[ update_index ]

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( serialisable_metadata, self._next_update_due ) = serialisable_info

		self._metadata = {}

		for ( update_index, encoded_update_hashes, begin, end ) in serialisable_metadata:
			update_hashes = [ bytes.fromhex( encoded_update_hash ) for encoded_update_hash in encoded_update_hashes ]

			self._metadata[ update_index ] = ( update_hashes, begin, end )

			self._update_hashes.update( update_hashes )

	def AppendUpdate( self, update_hashes, begin, end, next_update_due ):
		with self._lock:
			update_index = len( self._metadata )

			self._metadata[ update_index ] = ( update_hashes, begin, end )

			self._update_hashes.update( update_hashes )

			self._next_update_due = next_update_due

	def GetNextUpdateIndex( self ):
		with self._lock:
			return len( self._metadata )

	def GetNextUpdateBegin( self ):
		with self._lock:
			largest_update_index = max( self._metadata.keys() )

			( update_hashes, begin, end ) = self._GetUpdate( largest_update_index )

			return end + 1

	def GetNextUpdateDueString( self, from_client = False ):
		with self._lock:
			if self._next_update_due == 0:
				return 'have not yet synced metadata'

			else:
				update_due = self._GetNextUpdateDueTime( from_client )

				if TimeHasPassed( update_due ):
					s = 'imminently'

				else:
					s = TimestampToPrettyTimeDelta( update_due )

				return 'next update due ' + s

	def GetNumUpdateHashes( self ):
		with self._lock:
			num_update_hashes = sum( ( len( update_hashes ) for ( update_hashes, begin, end ) in list( self._metadata.values() ) ) )

			return num_update_hashes

	def GetSlice( self, from_update_index ):
		with self._lock:
			metadata = { update_index : row for ( update_index, row ) in list( self._metadata.items() ) if update_index >= from_update_index }

			return Metadata( metadata, self._next_update_due )

	def GetUpdateHashes( self, update_index = None ):
		with self._lock:
			if update_index is None:
				all_update_hashes = set()

				for ( update_hashes, begin, end ) in list( self._metadata.values() ):
					all_update_hashes.update( update_hashes )

				return all_update_hashes

			else:
				update_hashes = self._GetUpdateHashes( update_index )

				return update_hashes

	def GetUpdateIndicesAndHashes( self ):
		with self._lock:
			result = []

			for ( update_index, ( update_hashes, begin, end ) ) in list( self._metadata.items() ):
				result.append( ( update_index, update_hashes ) )

			return result

	def GetUpdateInfo( self, from_client = False ):
		with self._lock:
			num_update_hashes = sum( ( len( update_hashes ) for ( update_hashes, begin, end ) in list( self._metadata.values() ) ) )

			if len( self._metadata ) == 0:
				status = 'have not yet synchronised'

			else:
				biggest_end = max( ( end for ( update_hashes, begin, end ) in list( self._metadata.values() ) ) )

				delay = 0

				if from_client:
					delay = self.CLIENT_DELAY

				next_update_time = self._next_update_due + delay

				if TimeHasPassed( next_update_time ):
					s = 'imminently'

				else:
					s = TimestampToPrettyTimeDelta( next_update_time )

				status = 'metadata synchronised up to ' + TimestampToPrettyTimeDelta( biggest_end ) + ', next update due ' + s

			return ( num_update_hashes, status )

	def GetUpdateTimestamps( self, update_index ):
		with self._lock:
			update_timestamps = [ ( update_index, begin, end ) for ( update_index, ( update_hashes, begin, end ) ) in list( self._metadata.items() ) ]

			return update_timestamps

	def HasUpdateHash( self, update_hash ):
		with self._lock:
			return update_hash in self._update_hashes

	def UpdateDue( self, from_client = False ):
		with self._lock:
			next_update_due_time = self._GetNextUpdateDueTime( from_client )

			return TimeHasPassed( next_update_due_time )

	def UpdateFromSlice( self, metadata_slice ):
		with self._lock:
			self._metadata.update( metadata_slice._metadata )

			self._next_update_due = metadata_slice._next_update_due

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["METADATA"] ] = Metadata

class Petition( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["PETITION"]
	SERIALISABLE_NAME = 'Petition'
	SERIALISABLE_VERSION = 1

	def __init__( self, action = None, petitioner_account = None, reason = None, contents = None ):
		SerialisableBase.__init__( self )

		self._action = action
		self._petitioner_account = petitioner_account
		self._reason = reason
		self._contents = contents

	def _GetSerialisableInfo( self ):
		serialisable_petitioner_account = Account.GenerateSerialisableTupleFromAccount( self._petitioner_account )
		serialisable_contents = [ content.GetSerialisableTuple() for content in self._contents ]

		return ( self._action, serialisable_petitioner_account, self._reason, serialisable_contents )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self._action, serialisable_petitioner_account, self._reason, serialisable_contents ) = serialisable_info

		self._petitioner_account = Account.GenerateAccountFromSerialisableTuple( serialisable_petitioner_account )
		self._contents = [ CreateFromSerialisableTuple( serialisable_content ) for serialisable_content in serialisable_contents ]

	def GetActionTextAndColour( self ):
		action_text = ''

		if self._action == HC.CONTENT_UPDATE_PEND:
			action_text += 'ADD'
			action_colour = ( 127, 255, 127 )

		elif self._action == HC.CONTENT_UPDATE_PETITION:
			action_text += 'DELETE'
			action_colour = ( 255, 127, 127 )

		return ( action_text, action_colour )

	def GetApproval( self, contents ):
		if self._action == HC.CONTENT_UPDATE_PEND:
			content_update_action = HC.CONTENT_UPDATE_ADD

		elif self._action == HC.CONTENT_UPDATE_PETITION:
			content_update_action = HC.CONTENT_UPDATE_DELETE

		update = ClientToServerUpdate()
		content_updates = []

		for content in contents:
			update.AddContent( self._action, content, self._reason )

			content_type = content.GetContentType()

			row = content.GetContentData()

			content_update = ContentUpdate( content_type, content_update_action, row )

			content_updates.append( content_update )

		return ( update, content_updates )

	def GetContents( self ):
		return self._contents

	def GetDenial( self, contents ):
		if self._action == HC.CONTENT_UPDATE_PEND:
			denial_action = HC.CONTENT_UPDATE_DENY_PEND

		elif self._action == HC.CONTENT_UPDATE_PETITION:
			denial_action = HC.CONTENT_UPDATE_DENY_PETITION

		update = ClientToServerUpdate()

		for content in contents:
			update.AddContent( denial_action, content, self._reason )

		return update

	def GetPetitionerAccount( self ):
		return self._petitioner_account

	def GetReason( self ):
		return self._reason

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["PETITION"] ] = Petition

class ServerService( object ):
	def __init__( self, service_key, service_type, name, port, dictionary ):

		self._service_key = service_key
		self._service_type = service_type
		self._name = name
		self._port = port

		self._lock = threading.Lock()

		self._LoadFromDictionary( dictionary )

		self._dirty = False

	def _GetSerialisableDictionary( self ):
		dictionary = SerialisableDictionary()

		dictionary[ 'upnp_port' ] = self._upnp_port
		dictionary[ 'bandwidth_tracker' ] = self._bandwidth_tracker

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		self._upnp_port = dictionary[ 'upnp_port' ]
		self._bandwidth_tracker = dictionary[ 'bandwidth_tracker' ]

	def _SetDirty( self ):
		self._dirty = True

	def AllowsNonLocalConnections( self ):
		with self._lock:
			return True

	def BandwidthOK( self ):
		with self._lock:
			return True

	def Duplicate( self ):
		with self._lock:
			dictionary = self._GetSerialisableDictionary()

			dictionary = dictionary.Duplicate()

			duplicate = GenerateService( self._service_key, self._service_type, self._name, self._port, dictionary )

			return duplicate

	def GetName( self ):
		with self._lock:
			return self._name

	def GetPort( self ):
		with self._lock:
			return self._port

	def GetUPnPPort( self ):
		with self._lock:
			return self._upnp_port

	def GetServiceKey( self ):
		with self._lock:
			return self._service_key

	def GetServiceType( self ):
		with self._lock:
			return self._service_type

	def IsDirty( self ):
		with self._lock:
			return self._dirty

	def LogsRequests( self ):
		return True

	def ReportDataUsed( self, num_bytes ):
		with self._lock:
			self._bandwidth_tracker.ReportDataUsed( num_bytes )

			self._SetDirty()

	def ReportRequestUsed( self ):
		with self._lock:
			self._bandwidth_tracker.ReportRequestUsed()

			self._SetDirty()

	def SetClean( self ):
		with self._lock:
			self._dirty = False

	def SetName( self, name ):
		with self._lock:
			self._name = name

			self._SetDirty()

	def SetPort( self, port ):
		with self._lock:
			self._port = port

			self._SetDirty()

	def SupportsCORS( self ):
		return False

	def ToSerialisableTuple( self ):
		with self._lock:
			dictionary = self._GetSerialisableDictionary()

			dictionary_string = dictionary.DumpToString()

			return ( self._service_key.hex(), self._service_type, self._name, self._port, dictionary_string )

	def ToTuple( self ):
		with self._lock:
			dictionary = self._GetSerialisableDictionary()

			dictionary = dictionary.Duplicate()

			return ( self._service_key, self._service_type, self._name, self._port, dictionary )

class ServerServiceRestricted( ServerService ):
	def _GetSerialisableDictionary( self ):

		dictionary = ServerService._GetSerialisableDictionary( self )

		dictionary[ 'bandwidth_rules' ] = self._bandwidth_rules

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		ServerService._LoadFromDictionary( self, dictionary )

		self._bandwidth_rules = dictionary[ 'bandwidth_rules' ]

	def BandwidthOK( self ):
		with self._lock:
			return self._bandwidth_rules.CanStartRequest( self._bandwidth_tracker )

class ServerServiceRepository( ServerServiceRestricted ):
	def _GetSerialisableDictionary( self ):
		dictionary = ServerServiceRestricted._GetSerialisableDictionary( self )

		dictionary[ 'metadata' ] = self._metadata

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		ServerServiceRestricted._LoadFromDictionary( self, dictionary )

		self._metadata = dictionary[ 'metadata' ]

	def GetMetadataSlice( self, from_update_index ):
		with self._lock:
			return self._metadata.GetSlice( from_update_index )

	def HasUpdateHash( self, update_hash ):
		with self._lock:
			return self._metadata.HasUpdateHash( update_hash )

	def Sync( self ):
		with self._lock:
			update_due = self._metadata.UpdateDue()

		if update_due:
			HG.server_busy = True

			while update_due:
				with self._lock:
					service_key = self._service_key

					begin = self._metadata.GetNextUpdateBegin()

				end = begin + HC.UPDATE_DURATION

				update_hashes = HG.server_controller.WriteSynchronous( 'create_update', service_key, begin, end )

				next_update_due = end + HC.UPDATE_DURATION + 1

				with self._lock:
					self._metadata.AppendUpdate( update_hashes, begin, end, next_update_due )

					update_due = self._metadata.UpdateDue()

			HG.server_busy = False

			with self._lock:
				self._SetDirty()

class ServerServiceRepositoryTag( ServerServiceRepository ):
	pass

class ServerServiceRepositoryFile( ServerServiceRepository ):
	def _GetSerialisableDictionary( self ):
		dictionary = ServerServiceRepository._GetSerialisableDictionary( self )

		dictionary[ 'log_uploader_ips' ] = self._log_uploader_ips
		dictionary[ 'max_storage' ] = self._max_storage

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		ServerServiceRepository._LoadFromDictionary( self, dictionary )

		self._log_uploader_ips = dictionary[ 'log_uploader_ips' ]
		self._max_storage = dictionary[ 'max_storage' ]

	def LogUploaderIPs( self ):
		with self._lock:
			return self._log_uploader_ips

	def GetMaxStorage( self ):
		with self._lock:
			return self._max_storage

class ServerServiceAdmin( ServerServiceRestricted ):
	def _GetSerialisableDictionary( self ):
		dictionary = ServerServiceRestricted._GetSerialisableDictionary( self )

		dictionary[ 'server_bandwidth_tracker' ] = self._server_bandwidth_tracker
		dictionary[ 'server_bandwidth_rules' ] = self._server_bandwidth_rules

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		ServerServiceRestricted._LoadFromDictionary( self, dictionary )

		self._server_bandwidth_tracker = dictionary[ 'server_bandwidth_tracker' ]
		self._server_bandwidth_rules = dictionary[ 'server_bandwidth_rules' ]

	def ServerBandwidthOK( self ):
		with self._lock:
			return self._server_bandwidth_rules.CanStartRequest( self._server_bandwidth_tracker )

	def ServerReportDataUsed( self, num_bytes ):
		with self._lock:
			self._server_bandwidth_tracker.ReportDataUsed( num_bytes )

			self._SetDirty()

	def ServerReportRequestUsed( self ):
		with self._lock:
			self._server_bandwidth_tracker.ReportRequestUsed()

			self._SetDirty()

class UpdateBuilder( object ):
	def __init__( self, update_class, max_rows ):
		self._update_class = update_class
		self._max_rows = max_rows

		self._updates = []

		self._current_update = self._update_class()
		self._current_num_rows = 0

	def AddRow( self, row, row_weight = 1 ):
		self._current_update.AddRow( row )
		self._current_num_rows += row_weight

		if self._current_num_rows > self._max_rows:
			self._updates.append( self._current_update )

			self._current_update = self._update_class()
			self._current_num_rows = 0

	def Finish( self ):
		if self._current_update.GetNumRows() > 0:
			self._updates.append( self._current_update )

		self._current_update = None

	def GetUpdates( self ):
		return self._updates

license: wtfpl


code:
import threading, weakref

from lib.Hydrus.Exceptions import ShutdownException
from lib.Hydrus.Data import ShowText, ToHumanInt, Profile, ShowException

### CONSTANTS ###

import lib.Hydrus.Globals as HG

### CLASSES ###

class HydrusPubSub( object ):
	def __init__( self, controller ):
		self._controller = controller

		self._doing_work = False

		self._pubsubs = []

		self._pub_event = threading.Event()

		self._lock = threading.Lock()

		self._topics_to_objects = {}
		self._topics_to_method_names = {}

	def _GetCallables( self, topic ):
		callables = []

		if topic in self._topics_to_objects:
			try:
				objects = self._topics_to_objects[ topic ]

				for object in objects:
					if not object:
						continue

					method_names = self._topics_to_method_names[ topic ]

					for method_name in method_names:
						if hasattr( object, method_name ):
							callable = getattr( object, method_name )

							callables.append( callable )

			except Exception:
				pass

		return callables

	def DoingWork( self ):
		return self._doing_work

	def Process( self ):
		# only do one list of callables at a time
		# we don't want to map a topic to its callables until the previous topic's callables have been fully executed
		# e.g. when we start a message with a pubsub, it'll take a while (in independant thread-time) for wx to create
		# the dialog and hence map the new callable to the topic. this was leading to messages not being updated
		# because the (short) processing thread finished and entirely pubsubbed before wx had a chance to boot the
		# message.

		self._doing_work = True

		try:
			callables = []

			with self._lock:
				if len( self._pubsubs ) == 0:
					return

				pubsubs = self._pubsubs

				self._pubsubs = []

			for ( topic, args, kwargs ) in pubsubs:
				try:
					# do all this _outside_ the lock, lol

					callables = self._GetCallables( topic )

					# don't want to report the showtext we just send here!
					not_a_report = topic != 'message'

					if HG.pubsub_report_mode and not_a_report:
						ShowText( ( topic, args, kwargs, callables ) )

					if HG.pubsub_profile_mode and not_a_report:
						summary = 'Profiling ' + ToHumanInt( len( callables ) ) + ' x ' + topic

						ShowText( summary )

						per_summary = 'Profiling ' + topic

						for callable in callables:
							try:
								Profile( per_summary, 'callable( *args, **kwargs )', globals(), locals() )

							except ShutdownException:
								return False

					else:
						for callable in callables:
							try:
								callable( *args, **kwargs )

							except ShutdownException:
								return False

				except Exception as e:
					ShowException( e )

		finally:
			self._doing_work = False

	def pub( self, topic, *args, **kwargs ):
		with self._lock:
			self._pubsubs.append( ( topic, args, kwargs ) )

		self._pub_event.set()

	def pubimmediate( self, topic, *args, **kwargs ):
		with self._lock:
			callables = self._GetCallables( topic )

		for callable in callables:
			callable( *args, **kwargs )

	def sub( self, object, method_name, topic ):
		with self._lock:
			if topic not in self._topics_to_objects:
				self._topics_to_objects[ topic ] = weakref.WeakSet()
			if topic not in self._topics_to_method_names:
				self._topics_to_method_names[ topic ] = set()

			self._topics_to_objects[ topic ].add( object )
			self._topics_to_method_names[ topic ].add( method_name )

	def WaitOnPub( self ):
		self._pub_event.wait( 3 )

		self._pub_event.clear()

	def WorkToDo( self ):
		with self._lock:
			return len( self._pubsubs ) > 0


code:
import requests, pickle, threading

from lib.Hydrus.Data import GetNow, TimeHasPassed
from lib.Hydrus.Serialisable import CreateFromSerialisableTuple, SerialisableBase

from lib.Networking.Domain import ConvertDomainIntoSecondLevelDomain
from lib.Networking.Contexts import NetworkContext

try:
	import socket
	import socks

	SOCKS_PROXY_OK = True

except Exception:
	SOCKS_PROXY_OK = False

### CONSTANTS ###

import lib.Hydrus.Constants.Local as CC
import lib.Hydrus.Globals as HG

from lib.Hydrus.Serialisable import HG.SerialisableType["NETWORK_SESSION_MANAGER"], \
	SERIALISABLE_TYPES_TO_OBJECT_TYPES

### CLASSES ###

class NetworkSessionManager( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["NETWORK_SESSION_MANAGER"]
	SERIALISABLE_NAME = 'Session Manager'
	SERIALISABLE_VERSION = 1

	SESSION_TIMEOUT = 60 * 60

	def __init__( self ):
		SerialisableBase.__init__( self )

		self.engine = None

		self._dirty = False

		self._lock = threading.Lock()

		self._network_contexts_to_sessions = {}

		self._network_contexts_to_session_timeouts = {}

		self._proxies_dict = {}

		self._Reinitialise()

		HG.client_controller.sub( self, 'Reinitialise', 'notify_new_options' )

	def _CleanSessionCookies( self, network_context, session ):
		if network_context not in self._network_contexts_to_session_timeouts:
			self._network_contexts_to_session_timeouts[ network_context ] = 0

		if TimeHasPassed( self._network_contexts_to_session_timeouts[ network_context ] ):
			session.cookies.clear_session_cookies()

		self._network_contexts_to_session_timeouts[ network_context ] = GetNow() + self.SESSION_TIMEOUT

		session.cookies.clear_expired_cookies()

	def _GenerateSession( self, network_context ):
		session = requests.Session()

		if network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
			session.verify = False

		return session

	def _GetSerialisableInfo( self ):
		serialisable_network_contexts_to_sessions = [ ( network_context.GetSerialisableTuple(), pickle.dumps( session ).hex() ) for ( network_context, session ) in list(self._network_contexts_to_sessions.items() ) ]

		return serialisable_network_contexts_to_sessions

	def _GetSessionNetworkContext( self, network_context ):
		# just in case one of these slips through somehow
		if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
			second_level_domain = ConvertDomainIntoSecondLevelDomain( network_context.context_data )

			network_context = NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, second_level_domain )

		return network_context

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		serialisable_network_contexts_to_sessions = serialisable_info

		for ( serialisable_network_context, pickled_session_hex ) in serialisable_network_contexts_to_sessions:
			network_context = CreateFromSerialisableTuple( serialisable_network_context )

			try:
				session = pickle.loads( bytes.fromhex( pickled_session_hex ) )

			except Exception:
				# new version of requests uses a diff format, wew

				continue

			session.cookies.clear_session_cookies()

			self._network_contexts_to_sessions[ network_context ] = session

	def _Reinitialise( self ):
		self._proxies_dict = {}

		http_proxy = HG.client_controller.new_options.GetNoneableString( 'http_proxy' )
		https_proxy = HG.client_controller.new_options.GetNoneableString( 'https_proxy' )

		if http_proxy is not None:
			self._proxies_dict[ 'http' ] = http_proxy

		if https_proxy is not None:
			self._proxies_dict[ 'https' ] = https_proxy

	def _SetDirty( self ):
		self._dirty = True

	def ClearSession( self, network_context ):
		with self._lock:
			network_context = self._GetSessionNetworkContext( network_context )

			if network_context in self._network_contexts_to_sessions:
				del self._network_contexts_to_sessions[ network_context ]

				self._SetDirty()

	def GetNetworkContexts( self ):
		with self._lock:
			return list( self._network_contexts_to_sessions.keys() )

	def GetSession( self, network_context ):
		with self._lock:
			network_context = self._GetSessionNetworkContext( network_context )

			if network_context not in self._network_contexts_to_sessions:
				self._network_contexts_to_sessions[ network_context ] = self._GenerateSession( network_context )

			session = self._network_contexts_to_sessions[ network_context ]

			if session.proxies != self._proxies_dict:
				session.proxies = dict( self._proxies_dict )

			#

			self._CleanSessionCookies( network_context, session )

			#

			# tumblr can't into ssl for some reason, and the data subdomain they use has weird cert properties, looking like amazon S3
			# perhaps it is inward-facing somehow? whatever the case, let's just say fuck it for tumblr

			if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN and network_context.context_data == 'tumblr.com':
				session.verify = False

			#

			self._SetDirty()

			return session

	def GetSessionForDomain( self, domain ):
		network_context = NetworkContext( context_type = CC.NETWORK_CONTEXT_DOMAIN, context_data = domain )

		return self.GetSession( network_context )

	def IsDirty( self ):
		with self._lock:
			return self._dirty

	def Reinitialise( self ):
		with self._lock:
			self._Reinitialise()

	def SetClean( self ):
		with self._lock:
			self._dirty = False

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["NETWORK_SESSION_MANAGER"] ] = NetworkSessionManager


code:
from .Networking import *

__all__ = [
	"Bandwidth",
	"Contexts",
	"Domain",
	"Downloading",
	"Jobs",
	"Local",
	"Login",
	"NATPunch",
	"PubSub",
	"Sessions"
]

from . import *


code:
import lib.Hydrus.Constants as HC
import os

def do_2to3_test( wx_error_display_callable = None ):
	bad_filenames = [ 'python27.dll', 'lz4._version.so' ]

	for bad_filename in bad_filenames:
		path = os.path.join( HC.BASE_DIR, bad_filename )

		if os.path.exists( path ):
			message = 'It looks like you still have some Python 2 files in your install directory! Hydrus is now Python 3 and needs a clean install. Please check the v335 release post for more information! The program will now exit!'

			if wx_error_display_callable is not None:
				wx_error_display_callable( 'Python 2/3 Error!', message )

			print( message )

			raise Exception( 'Client needs a clean install!' )