code:
### IMPORTS ###
from lib.Client import Caches, ClientConstants as CC, API, LocalServer, LocalServerResources, Media, Ratings, Search, ClientServices, ClientTags

from lib.Hydrus import Constants as HC, Exceptions, Tags, Text, Globals as HG, Data

import collections, hashlib, http.client, json, os, random, shutil, time, unittest, urllib

from twisted.internet import reactor

### CODE ###

class TestClientAPI( unittest.TestCase ):
	@classmethod

	def setUpClass( cls ):
		cls._client_api = lib.Hydrus.Services.Local.GenerateService( CC.CLIENT_API_SERVICE_KEY, HC.CLIENT_API_SERVICE, 'client api' )
		cls._client_api_cors = lib.Hydrus.Services.Local.GenerateService( CC.CLIENT_API_SERVICE_KEY, HC.CLIENT_API_SERVICE, 'client api' )

		cls._client_api_cors._support_cors = True

		def TWISTEDSetup():
			reactor.listenTCP( 45869, LocalServer.HydrusServiceClientAPI( cls._client_api, allow_non_local_connections = False ) )
			reactor.listenTCP( 45899, LocalServer.HydrusServiceClientAPI( cls._client_api_cors, allow_non_local_connections = False ) )

		reactor.callFromThread( TWISTEDSetup )

		time.sleep( 1 )

	def _test_basics( self, connection ):
		#

		connection.request( 'GET', '/' )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		#

		with open( os.path.join( HC.STATIC_DIR, 'hydrus.ico' ), 'rb' ) as f:
			favicon = f.read()

		connection.request( 'GET', '/favicon.ico' )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( data, favicon )

	def _test_client_api_basics( self, connection ):
		# /api_version

		connection.request( 'GET', '/api_version' )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		response_json = json.loads( text )

		self.assertEqual( response_json[ 'version' ], HC.CLIENT_API_VERSION )

		# /request_new_permissions

		def format_request_new_permissions_query( name, basic_permissions ):
			return '/request_new_permissions?name={}&basic_permissions={}'.format( urllib.parse.quote( name ), urllib.parse.quote( json.dumps( basic_permissions ) ) )

		# fail

		API.api_request_dialog_open = False

		connection.request( 'GET', format_request_new_permissions_query( 'test', [ API.CLIENT_API_PERMISSION["ADD_FILES"] ] ) )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		self.assertIn( 'dialog', text )

		self.assertEqual( response.status, 403 )

		# success

		permissions_to_set_up = []

		permissions_to_set_up.append( ( 'everything', list( API.ALLOWED_PERMISSIONS ) ) )
		permissions_to_set_up.append( ( 'add_files', [ API.CLIENT_API_PERMISSION["ADD_FILES"] ] ) )
		permissions_to_set_up.append( ( 'add_tags', [ API.CLIENT_API_PERMISSION["ADD_TAGS"] ] ) )
		permissions_to_set_up.append( ( 'add_urls', [ API.CLIENT_API_PERMISSION["ADD_URLS"] ] ) )
		permissions_to_set_up.append( ( 'search_all_files', [ API.CLIENT_API_PERMISSION["SEARCH_FILES"] ] ) )
		permissions_to_set_up.append( ( 'search_green_files', [ API.CLIENT_API_PERMISSION["SEARCH_FILES"] ] ) )

		set_up_permissions = {}

		for ( name, basic_permissions ) in permissions_to_set_up:
			API.api_request_dialog_open = True

			connection.request( 'GET', format_request_new_permissions_query( name, basic_permissions ) )

			response = connection.getresponse()

			data = response.read()

			API.api_request_dialog_open = False

			self.assertEqual( response.status, 200 )

			response_text = str( data, 'utf-8' )

			response_json = json.loads( response_text )

			access_key_hex = response_json[ 'access_key' ]

			self.assertEqual( len( access_key_hex ), 64 )

			access_key_hex = Text.HexFilter( access_key_hex )

			self.assertEqual( len( access_key_hex ), 64 )

			api_permissions = API.last_api_permissions_request

			if 'green' in name:
				search_tag_filter = lib.Hydrus.Tags.Local.TagFilter()

				search_tag_filter.SetRule( '', CC.FILTER_BLACKLIST )
				search_tag_filter.SetRule( ':', CC.FILTER_BLACKLIST )
				search_tag_filter.SetRule( 'green', CC.FILTER_WHITELIST )

				api_permissions.SetSearchTagFilter( search_tag_filter )

			self.assertEqual( bytes.fromhex( access_key_hex ), api_permissions.GetAccessKey() )

			set_up_permissions[ name ] = api_permissions

			HG.test_controller.client_api_manager.AddAccess( api_permissions )

		# /verify_access_key

		# missing

		connection.request( 'GET', '/verify_access_key' )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 401 )

		# fail

		incorrect_headers = { 'Hydrus-Client-API-Access-Key' : 'abcd' }

		connection.request( 'GET', '/verify_access_key', headers = incorrect_headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 403 )

		# success

		for ( name, api_permissions ) in set_up_permissions.items():
			access_key_hex = api_permissions.GetAccessKey().hex()

			headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex }

			connection.request( 'GET', '/verify_access_key', headers = headers )

			response = connection.getresponse()

			data = response.read()

			text = str( data, 'utf-8' )

			body_dict = json.loads( text )

			self.assertEqual( response.status, 200 )

			self.assertEqual( set( body_dict[ 'basic_permissions' ] ), set( api_permissions.GetBasicPermissions() ) )
			self.assertEqual( body_dict[ 'human_description' ], api_permissions.ToHumanString() )

		return set_up_permissions

	def _test_cors_fails( self, connection ):
		connection.request( 'OPTIONS', '/api_version' )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 204 )

		self.assertEqual( response.getheader( 'Allow' ), 'GET' )

		#

		connection.request( 'OPTIONS', '/api_version', headers = { 'Origin' : 'muhsite.com' } )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 401 )

	def _test_cors_succeeds( self, connection ):
		connection.request( 'OPTIONS', '/api_version' )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 204 )

		self.assertEqual( response.getheader( 'Allow' ), 'GET' )

		#

		connection.request( 'OPTIONS', '/api_version', headers = { 'Origin' : 'muhsite.com' } )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 204 )

		self.assertEqual( response.getheader( 'Access-Control-Allow-Methods' ), 'GET' )
		self.assertEqual( response.getheader( 'Access-Control-Allow-Origin' ), '*' )

	def _test_add_files( self, connection, set_up_permissions ):
		api_permissions = set_up_permissions[ 'add_files' ]

		access_key_hex = api_permissions.GetAccessKey().hex()

		# fail

		HG.test_controller.SetRead( 'hash_status', ( CC.STATUS_UNKNOWN, None, '' ) )

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex, 'Content-Type' : HC.mime_string_lookup[ HC.APPLICATION_OCTET_STREAM ] }

		path = '/add_files/add_file'

		body = b'blarg'

		connection.request( 'POST', path, body = body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		self.assertEqual( response.status, 200 )

		response_json = json.loads( text )

		self.assertEqual( response_json[ 'status' ], CC.STATUS_ERROR )
		self.assertEqual( response_json[ 'hash' ], 'a593942cb7ea9ffcd8ccf2f0fa23c338e23bfecd9a3e508dfc0bcf07501ead08' )
		self.assertIn( 'Traceback', response_json[ 'note' ] )

		# success as body

		hydrus_png_path = os.path.join( HC.STATIC_DIR, 'hydrus.png' )

		with open( hydrus_png_path, 'rb' ) as f:
			HYDRUS_PNG_BYTES = f.read()

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex, 'Content-Type' : HC.mime_string_lookup[ HC.APPLICATION_OCTET_STREAM ] }

		path = '/add_files/add_file'

		body = HYDRUS_PNG_BYTES

		connection.request( 'POST', path, body = body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		self.assertEqual( response.status, 200 )

		response_json = json.loads( text )

		expected_result = { 'status' : CC.STATUS_SUCCESSFUL_AND_NEW, 'hash' : 'ad6d3599a6c489a575eb19c026face97a9cd6579e74728b0ce94a601d232f3c3' , 'note' : 'test note' }

		self.assertEqual( response_json, expected_result )

		# do hydrus png as path

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex, 'Content-Type' : HC.mime_string_lookup[ HC.APPLICATION_JSON ] }

		path = '/add_files/add_file'

		body_dict = { 'path' : hydrus_png_path }

		body = json.dumps( body_dict )

		connection.request( 'POST', path, body = body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		self.assertEqual( response.status, 200 )

		response_json = json.loads( text )

		expected_result = { 'status' : CC.STATUS_SUCCESSFUL_AND_NEW, 'hash' : 'ad6d3599a6c489a575eb19c026face97a9cd6579e74728b0ce94a601d232f3c3' , 'note' : 'test note' }

		self.assertEqual( response_json, expected_result )

	def _test_add_tags( self, connection, set_up_permissions ):
		# get services

		api_permissions = set_up_permissions[ 'everything' ]

		access_key_hex = api_permissions.GetAccessKey().hex()

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex }

		#

		path = '/add_tags/get_tag_services'

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		self.assertEqual( response.status, 200 )

		d = json.loads( text )

		expected_answer = {}

		expected_answer[ 'local_tags' ] = [ "local tags" ]
		expected_answer[ 'tag_repositories' ] = [ "example tag repo" ]

		self.assertEqual( d, expected_answer )

		# clean tags

		tags = [ " bikini ", "blue	eyes", " character : samus aran ", ":)", "   ", "", "10", "11", "9", "system:wew", "-flower" ]

		json_tags = json.dumps( tags )

		path = '/add_tags/clean_tags?tags={}'.format( urllib.parse.quote( json_tags, safe = '' ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		self.assertEqual( response.status, 200 )

		d = json.loads( text )

		expected_answer = {}

		clean_tags = [ "bikini", "blue eyes", "character:samus aran", "::)", "10", "11", "9", "wew", "flower" ]

		clean_tags = Tags.SortNumericTags( clean_tags )

		expected_answer[ 'tags' ] = clean_tags

		self.assertEqual( d, expected_answer )

		# add tags

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex, 'Content-Type' : HC.mime_string_lookup[ HC.APPLICATION_JSON ] }

		hash = os.urandom( 32 )
		hash_hex = hash.hex()

		hash2 = os.urandom( 32 )
		hash2_hex = hash2.hex()

		# missing hashes

		path = '/add_tags/add_tags'

		body_dict = { 'service_names_to_tags' : { 'local tags' : [ 'test' ] } }

		body = json.dumps( body_dict )

		connection.request( 'POST', path, body = body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 400 )

		# invalid service key

		path = '/add_tags/add_tags'

		body_dict = { 'hash' : hash_hex, 'service_names_to_tags' : { 'bad tag service' : [ 'test' ] } }

		body = json.dumps( body_dict )

		connection.request( 'POST', path, body = body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 400 )

		# add tags to local

		HG.test_controller.ClearWrites( 'content_updates' )

		path = '/add_tags/add_tags'

		body_dict = { 'hash' : hash_hex, 'service_names_to_tags' : { 'local tags' : [ 'test', 'test2' ] } }

		body = json.dumps( body_dict )

		connection.request( 'POST', path, body = body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		expected_service_keys_to_content_updates = collections.defaultdict( list )

		expected_service_keys_to_content_updates[ CC.LOCAL_TAG_SERVICE_KEY ] = [ Data.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'test', set( [ hash ] ) ) ), Data.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'test2', set( [ hash ] ) ) ) ]

		[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )

		self.assertEqual( len( service_keys_to_content_updates ), len( expected_service_keys_to_content_updates ) )

		for ( service_key, content_updates ) in service_keys_to_content_updates.items():
			expected_content_updates = expected_service_keys_to_content_updates[ service_key ]

			c_u_tuples = [ c_u.ToTuple() for c_u in content_updates ]
			e_c_u_tuples = [ e_c_u.ToTuple() for e_c_u in expected_content_updates ]

			c_u_tuples.sort()
			e_c_u_tuples.sort()

			self.assertEqual( c_u_tuples, e_c_u_tuples )

		# add to multiple files

		HG.test_controller.ClearWrites( 'content_updates' )

		path = '/add_tags/add_tags'

		body_dict = { 'hashes' : [ hash_hex, hash2_hex ], 'service_names_to_tags' : { 'local tags' : [ 'test', 'test2' ] } }

		body = json.dumps( body_dict )

		connection.request( 'POST', path, body = body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		expected_service_keys_to_content_updates = collections.defaultdict( list )

		expected_service_keys_to_content_updates[ CC.LOCAL_TAG_SERVICE_KEY ] = [ Data.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'test', set( [ hash, hash2 ] ) ) ), Data.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'test2', set( [ hash, hash2 ] ) ) ) ]

		[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )

		self.assertEqual( len( service_keys_to_content_updates ), len( expected_service_keys_to_content_updates ) )

		for ( service_key, content_updates ) in service_keys_to_content_updates.items():
			expected_content_updates = expected_service_keys_to_content_updates[ service_key ]

			c_u_tuples = [ c_u.ToTuple() for c_u in content_updates ]
			e_c_u_tuples = [ e_c_u.ToTuple() for e_c_u in expected_content_updates ]

			c_u_tuples.sort()
			e_c_u_tuples.sort()

			self.assertEqual( c_u_tuples, e_c_u_tuples )

		# siblings and parents

		# setting up

		old_sib = HG.test_controller.tag_siblings_manager
		old_par = HG.test_controller.tag_parents_manager

		tag_siblings = collections.defaultdict( Data.default_dict_set )

		first_dict = Data.default_dict_set()

		first_dict[ HC.CONTENT_STATUS_CURRENT ] = { ( 'test', 'muh test' ) }

		tag_siblings[ CC.LOCAL_TAG_SERVICE_KEY ] = first_dict

		HG.test_controller.SetRead( 'tag_siblings', tag_siblings )

		tag_parents = collections.defaultdict( Data.default_dict_set )

		first_dict = Data.default_dict_set()

		first_dict[ HC.CONTENT_STATUS_CURRENT ] = { ( 'muh test', 'muh test parent' ) }

		tag_parents[ CC.LOCAL_TAG_SERVICE_KEY ] = first_dict

		HG.test_controller.SetRead( 'tag_parents', tag_parents )

		HG.test_controller.tag_siblings_manager = Caches.TagSiblingsManager( HG.test_controller )
		HG.test_controller.tag_parents_manager = Caches.TagParentsManager( HG.test_controller )

		# ok, now with

		HG.test_controller.ClearWrites( 'content_updates' )

		path = '/add_tags/add_tags'

		body_dict = { 'hash' : hash_hex, 'service_names_to_tags' : { 'local tags' : [ 'test' ] } }

		body = json.dumps( body_dict )

		connection.request( 'POST', path, body = body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		expected_service_keys_to_content_updates = collections.defaultdict( list )

		expected_service_keys_to_content_updates[ CC.LOCAL_TAG_SERVICE_KEY ] = [ Data.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'muh test', set( [ hash ] ) ) ), Data.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'muh test parent', set( [ hash ] ) ) ) ]

		[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )

		self.assertEqual( len( service_keys_to_content_updates ), len( expected_service_keys_to_content_updates ) )

		for ( service_key, content_updates ) in service_keys_to_content_updates.items():
			expected_content_updates = expected_service_keys_to_content_updates[ service_key ]

			c_u_tuples = [ c_u.ToTuple() for c_u in content_updates ]
			e_c_u_tuples = [ e_c_u.ToTuple() for e_c_u in expected_content_updates ]

			c_u_tuples.sort()
			e_c_u_tuples.sort()

			self.assertEqual( c_u_tuples, e_c_u_tuples )

		# and without

		HG.test_controller.ClearWrites( 'content_updates' )

		path = '/add_tags/add_tags'

		body_dict = { 'hash' : hash_hex, 'service_names_to_tags' : { 'local tags' : [ 'test' ] }, 'add_siblings_and_parents' : False }

		body = json.dumps( body_dict )

		connection.request( 'POST', path, body = body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		expected_service_keys_to_content_updates = collections.defaultdict( list )

		expected_service_keys_to_content_updates[ CC.LOCAL_TAG_SERVICE_KEY ] = [ Data.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'test', set( [ hash ] ) ) ) ]

		[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )

		self.assertEqual( len( service_keys_to_content_updates ), len( expected_service_keys_to_content_updates ) )

		for ( service_key, content_updates ) in service_keys_to_content_updates.items():
			expected_content_updates = expected_service_keys_to_content_updates[ service_key ]

			c_u_tuples = [ c_u.ToTuple() for c_u in content_updates ]
			e_c_u_tuples = [ e_c_u.ToTuple() for e_c_u in expected_content_updates ]

			c_u_tuples.sort()
			e_c_u_tuples.sort()

			self.assertEqual( c_u_tuples, e_c_u_tuples )

		# cleanup

		HG.test_controller.tag_siblings_manager = old_sib
		HG.test_controller.tag_parents_manager = old_par

	def _test_add_urls( self, connection, set_up_permissions ):
		# get url files

		api_permissions = set_up_permissions[ 'everything' ]

		access_key_hex = api_permissions.GetAccessKey().hex()

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex }

		# none

		url = 'https://muhsite.wew/help_compute'

		HG.test_controller.SetRead( 'url_statuses', [] )

		path = '/add_urls/get_url_files?url={}'.format( urllib.parse.quote( url, safe = '' ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		self.assertEqual( response.status, 200 )

		d = json.loads( text )

		expected_answer = {}

		expected_answer[ 'normalised_url' ] = url
		expected_answer[ 'url_file_statuses' ] = []

		self.assertEqual( d, expected_answer )

		# some

		url = 'http://safebooru.org/index.php?page=post&s=view&id=2753608'
		normalised_url = 'https://safebooru.org/index.php?id=2753608&page=post&s=view'

		hash = os.urandom( 32 )

		url_file_statuses = [ ( CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, hash, 'muh import phrase' ) ]
		json_url_file_statuses = [ { 'status' : CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, 'hash' : hash.hex(), 'note' : 'muh import phrase' } ]

		HG.test_controller.SetRead( 'url_statuses', url_file_statuses )

		path = '/add_urls/get_url_files?url={}'.format( urllib.parse.quote( url, safe = '' ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		self.assertEqual( response.status, 200 )

		d = json.loads( text )

		expected_answer = {}

		expected_answer[ 'normalised_url' ] = normalised_url
		expected_answer[ 'url_file_statuses' ] = json_url_file_statuses

		self.assertEqual( d, expected_answer )

		# get url info

		api_permissions = set_up_permissions[ 'everything' ]

		access_key_hex = api_permissions.GetAccessKey().hex()

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex }

		# unknown

		url = 'https://muhsite.wew/help_compute'

		path = '/add_urls/get_url_info?url={}'.format( urllib.parse.quote( url, safe = '' ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		self.assertEqual( response.status, 200 )

		d = json.loads( text )

		expected_answer = {}

		expected_answer[ 'normalised_url' ] = url
		expected_answer[ 'url_type' ] = HC.URL_TYPE_UNKNOWN
		expected_answer[ 'url_type_string' ] = 'unknown url'
		expected_answer[ 'match_name' ] = 'unknown url'
		expected_answer[ 'can_parse' ] = False

		self.assertEqual( d, expected_answer )

		# known

		url = 'http://8ch.net/tv/res/1846574.html'
		normalised_url = 'https://8ch.net/tv/res/1846574.html'
		# http so we can test normalised is https

		path = '/add_urls/get_url_info?url={}'.format( urllib.parse.quote( url, safe = '' ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		self.assertEqual( response.status, 200 )

		d = json.loads( text )

		expected_answer = {}

		expected_answer = { 'url_type' : HC.URL_TYPE_WATCHABLE, 'url_type_string' : 'watchable url', 'match_name' : '8chan thread', 'can_parse' : True }

		expected_answer[ 'normalised_url' ] = normalised_url
		expected_answer[ 'url_type' ] = HC.URL_TYPE_WATCHABLE
		expected_answer[ 'url_type_string' ] = 'watchable url'
		expected_answer[ 'match_name' ] = '8chan thread'
		expected_answer[ 'can_parse' ] = True

		self.assertEqual( d, expected_answer )

		# known post url

		url = 'http://safebooru.org/index.php?page=post&s=view&id=2753608'
		normalised_url = 'https://safebooru.org/index.php?id=2753608&page=post&s=view'

		hash = os.urandom( 32 )

		path = '/add_urls/get_url_info?url={}'.format( urllib.parse.quote( url, safe = '' ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		text = str( data, 'utf-8' )

		self.assertEqual( response.status, 200 )

		d = json.loads( text )

		expected_answer = {}

		expected_answer[ 'normalised_url' ] = normalised_url
		expected_answer[ 'url_type' ] = HC.URL_TYPE_POST
		expected_answer[ 'url_type_string' ] = 'post url'
		expected_answer[ 'match_name' ] = 'safebooru file page'
		expected_answer[ 'can_parse' ] = True

		self.assertEqual( d, expected_answer )

		# add url

		HG.test_controller.ClearWrites( 'import_url_test' )

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex, 'Content-Type' : HC.mime_string_lookup[ HC.APPLICATION_JSON ] }

		url = 'http://8ch.net/tv/res/1846574.html'

		request_dict = { 'url' : url }

		request_body = json.dumps( request_dict )

		connection.request( 'POST', '/add_urls/add_url', body = request_body, headers = headers )

		response = connection.getresponse()

		self.assertEqual( response.status, 200 )

		data = response.read()

		text = str( data, 'utf-8' )

		response_json = json.loads( text )

		self.assertEqual( response_json[ 'human_result_text' ], '"https://8ch.net/tv/res/1846574.html" URL added successfully.' )
		self.assertEqual( response_json[ 'normalised_url' ], 'https://8ch.net/tv/res/1846574.html' )

		self.assertEqual( HG.test_controller.GetWrite( 'import_url_test' ), [ ( ( url, None, None, False ), {} ) ] )

		# with name

		HG.test_controller.ClearWrites( 'import_url_test' )

		request_dict = { 'url' : url, 'destination_page_name' : 'muh /tv/' }

		request_body = json.dumps( request_dict )

		connection.request( 'POST', '/add_urls/add_url', body = request_body, headers = headers )

		response = connection.getresponse()

		self.assertEqual( response.status, 200 )

		data = response.read()

		text = str( data, 'utf-8' )

		response_json = json.loads( text )

		self.assertEqual( response_json[ 'human_result_text' ], '"https://8ch.net/tv/res/1846574.html" URL added successfully.' )
		self.assertEqual( response_json[ 'normalised_url' ], 'https://8ch.net/tv/res/1846574.html' )

		self.assertEqual( HG.test_controller.GetWrite( 'import_url_test' ), [ ( ( url, None, 'muh /tv/', False ), {} ) ] )

		# add tags and name, and show destination page

		HG.test_controller.ClearWrites( 'import_url_test' )

		request_dict = { 'url' : url, 'destination_page_name' : 'muh /tv/', 'show_destination_page' : True, 'service_names_to_tags' : { 'local tags' : [ '/tv/ thread' ] } }

		request_body = json.dumps( request_dict )

		connection.request( 'POST', '/add_urls/add_url', body = request_body, headers = headers )

		response = connection.getresponse()

		self.assertEqual( response.status, 200 )

		data = response.read()

		text = str( data, 'utf-8' )

		response_json = json.loads( text )

		self.assertEqual( response_json[ 'human_result_text' ], '"https://8ch.net/tv/res/1846574.html" URL added successfully.' )
		self.assertEqual( response_json[ 'normalised_url' ], 'https://8ch.net/tv/res/1846574.html' )

		service_keys_to_tags = lib.Hydrus.Tags.Local.ServiceKeysToTags( { CC.LOCAL_TAG_SERVICE_KEY : set( [ '/tv/ thread' ] ) } )

		self.assertEqual( HG.test_controller.GetWrite( 'import_url_test' ), [ ( ( url, service_keys_to_tags, 'muh /tv/', True ), {} ) ] )

		# associate url

		HG.test_controller.ClearWrites( 'content_updates' )

		hash = bytes.fromhex( '3b820114f658d768550e4e3d4f1dced3ff8db77443472b5ad93700647ad2d3ba' )
		url = 'https://rule34.xxx/index.php?id=2588418&page=post&s=view'

		request_dict = { 'url_to_add' : url, 'hash' : hash.hex() }

		request_body = json.dumps( request_dict )

		connection.request( 'POST', '/add_urls/associate_url', body = request_body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		expected_service_keys_to_content_updates = collections.defaultdict( list )

		expected_service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ] = [ Data.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( [ url ], [ hash ] ) ) ]

		expected_result = [ ( ( expected_service_keys_to_content_updates, ), {} ) ]

		result = HG.test_controller.GetWrite( 'content_updates' )

		self.assertEqual( result, expected_result )

		#

		HG.test_controller.ClearWrites( 'content_updates' )

		hash = bytes.fromhex( '3b820114f658d768550e4e3d4f1dced3ff8db77443472b5ad93700647ad2d3ba' )
		url = 'https://rule34.xxx/index.php?id=2588418&page=post&s=view'

		request_dict = { 'urls_to_add' : [ url ], 'hashes' : [ hash.hex() ] }

		request_body = json.dumps( request_dict )

		connection.request( 'POST', '/add_urls/associate_url', body = request_body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		expected_service_keys_to_content_updates = collections.defaultdict( list )

		expected_service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ] = [ Data.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( [ url ], [ hash ] ) ) ]

		expected_result = [ ( ( expected_service_keys_to_content_updates, ), {} ) ]

		result = HG.test_controller.GetWrite( 'content_updates' )

		self.assertEqual( result, expected_result )

		#

		HG.test_controller.ClearWrites( 'content_updates' )

		hash = bytes.fromhex( '3b820114f658d768550e4e3d4f1dced3ff8db77443472b5ad93700647ad2d3ba' )
		url = 'http://rule34.xxx/index.php?id=2588418&page=post&s=view'

		request_dict = { 'url_to_delete' : url, 'hash' : hash.hex() }

		request_body = json.dumps( request_dict )

		connection.request( 'POST', '/add_urls/associate_url', body = request_body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		expected_service_keys_to_content_updates = collections.defaultdict( list )

		expected_service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ] = [ Data.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_DELETE, ( [ url ], [ hash ] ) ) ]

		expected_result = [ ( ( expected_service_keys_to_content_updates, ), {} ) ]

		result = HG.test_controller.GetWrite( 'content_updates' )

		self.assertEqual( result, expected_result )

		#

		HG.test_controller.ClearWrites( 'content_updates' )

		hash = bytes.fromhex( '3b820114f658d768550e4e3d4f1dced3ff8db77443472b5ad93700647ad2d3ba' )
		url = 'http://rule34.xxx/index.php?id=2588418&page=post&s=view'

		request_dict = { 'urls_to_delete' : [ url ], 'hashes' : [ hash.hex() ] }

		request_body = json.dumps( request_dict )

		connection.request( 'POST', '/add_urls/associate_url', body = request_body, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		expected_service_keys_to_content_updates = collections.defaultdict( list )

		expected_service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ] = [ Data.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_DELETE, ( [ url ], [ hash ] ) ) ]

		expected_result = [ ( ( expected_service_keys_to_content_updates, ), {} ) ]

		result = HG.test_controller.GetWrite( 'content_updates' )

		self.assertEqual( result, expected_result )

	def _test_search_files( self, connection, set_up_permissions ):
		hash_ids = [ 1, 2, 3, 4, 5, 10 ]

		HG.test_controller.SetRead( 'file_query_ids', set( hash_ids ) )

		# search files failed tag permission

		api_permissions = set_up_permissions[ 'search_green_files' ]

		access_key_hex = api_permissions.GetAccessKey().hex()

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex }

		#

		tags = []

		path = '/get_files/search_files?tags={}'.format( urllib.parse.quote( json.dumps( tags ) ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 403 )

		#

		tags = [ 'kino' ]

		path = '/get_files/search_files?tags={}'.format( urllib.parse.quote( json.dumps( tags ) ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 403 )

		# search files

		tags = [ 'kino', 'green' ]

		path = '/get_files/search_files?tags={}'.format( urllib.parse.quote( json.dumps( tags ) ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		text = str( data, 'utf-8' )

		d = json.loads( text )

		expected_answer = { 'file_ids' : hash_ids }

		self.assertEqual( d, expected_answer )

		# some file search param parsing

		class PretendRequest( object ):
			pass

		pretend_request = PretendRequest()

		pretend_request.parsed_request_args = {}
		pretend_request.client_api_permissions = set_up_permissions[ 'everything' ]

		predicates = LocalServerResources.ParseClientAPISearchPredicates( pretend_request )

		self.assertEqual( predicates, [] )

		#

		pretend_request = PretendRequest()

		pretend_request.parsed_request_args = { 'system_inbox' : True }
		pretend_request.client_api_permissions = set_up_permissions[ 'search_green_files' ]

		with self.assertRaises( Exceptions.InsufficientCredentialsException ):
			LocalServerResources.ParseClientAPISearchPredicates( pretend_request )

		#

		pretend_request = PretendRequest()

		pretend_request.parsed_request_args = { 'tags' : [ '-green' ] }
		pretend_request.client_api_permissions = set_up_permissions[ 'search_green_files' ]

		with self.assertRaises( Exceptions.InsufficientCredentialsException ):
			LocalServerResources.ParseClientAPISearchPredicates( pretend_request )

		#

		pretend_request = PretendRequest()

		pretend_request.parsed_request_args = { 'tags' : [ 'green', '-kino' ] }
		pretend_request.client_api_permissions = set_up_permissions[ 'search_green_files' ]

		predicates = LocalServerResources.ParseClientAPISearchPredicates( pretend_request )

		expected_predicates = []

		expected_predicates.append( Search.Predicate( predicate_type = HC.PREDICATE_TYPE_TAG, value = 'green' ) )
		expected_predicates.append( Search.Predicate( predicate_type = HC.PREDICATE_TYPE_TAG, value = 'kino', inclusive = False ) )

		self.assertEqual( set( predicates ), set( expected_predicates ) )

		#

		pretend_request = PretendRequest()

		pretend_request.parsed_request_args = { 'tags' : [ 'green' ], 'system_inbox' : True }
		pretend_request.client_api_permissions = set_up_permissions[ 'search_green_files' ]

		predicates = LocalServerResources.ParseClientAPISearchPredicates( pretend_request )

		expected_predicates = []

		expected_predicates.append( Search.Predicate( predicate_type = HC.PREDICATE_TYPE_TAG, value = 'green' ) )
		expected_predicates.append( Search.Predicate( predicate_type = HC.PREDICATE_TYPE_SYSTEM_INBOX ) )

		self.assertEqual( set( predicates ), set( expected_predicates ) )

		#

		pretend_request = PretendRequest()

		pretend_request.parsed_request_args = { 'tags' : [ 'green' ], 'system_archive' : True }
		pretend_request.client_api_permissions = set_up_permissions[ 'search_green_files' ]

		predicates = LocalServerResources.ParseClientAPISearchPredicates( pretend_request )

		expected_predicates = []

		expected_predicates.append( Search.Predicate( predicate_type = HC.PREDICATE_TYPE_TAG, value = 'green' ) )
		expected_predicates.append( Search.Predicate( predicate_type = HC.PREDICATE_TYPE_SYSTEM_ARCHIVE ) )

		self.assertEqual( set( predicates ), set( expected_predicates ) )

		# test file metadata

		api_permissions = set_up_permissions[ 'search_green_files' ]

		access_key_hex = api_permissions.GetAccessKey().hex()

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex }

		file_ids_to_hashes = { 1 : bytes.fromhex( 'a' * 64 ), 2 : bytes.fromhex( 'b' * 64 ), 3 : bytes.fromhex( 'c' * 64 ) }

		metadata = []

		for ( file_id, hash ) in file_ids_to_hashes.items():
			metadata_row = { 'file_id' : file_id, 'hash' : hash.hex() }

			metadata.append( metadata_row )

		expected_identifier_result = { 'metadata' :
	metadata }

		media_results = []

		for ( file_id, hash ) in file_ids_to_hashes.items():
			size = random.randint( 8192, 20 * 1048576 )
			mime = random.choice( [ HC.IMAGE_JPEG, HC.VIDEO_WEBM, HC.APPLICATION_PDF ] )
			width = random.randint( 200, 4096 )
			height = random.randint( 200, 4096 )
			duration = random.choice( [ 220, 16.66667, None ] )

			file_info_manager = Media.FileInfoManager( file_id, hash, size = size, mime = mime, width = width, height = height, duration = duration )

			service_keys_to_statuses_to_tags = { CC.LOCAL_TAG_SERVICE_KEY : { HC.CONTENT_STATUS_CURRENT : [ 'blue eyes', 'blonde hair' ], HC.CONTENT_STATUS_PENDING : [ 'bodysuit' ] } }

			tags_manager = Media.TagsManager( service_keys_to_statuses_to_tags )

			locations_manager = Media.LocationsManager( set(), set(), set(), set() )
			ratings_manager = Ratings.RatingsManager( {} )
			file_viewing_stats_manager = Media.FileViewingStatsManager( 0, 0, 0, 0 )

			media_result = Media.MediaResult( file_info_manager, tags_manager, locations_manager, ratings_manager, file_viewing_stats_manager )

			media_results.append( media_result )

		metadata = []

		services_manager = HG.client_controller.services_manager

		service_keys_to_names = {}

		for media_result in media_results:
			metadata_row = {}

			file_info_manager = media_result.GetFileInfoManager()

			metadata_row[ 'file_id' ] = file_info_manager.hash_id
			metadata_row[ 'hash' ] = file_info_manager.hash.hex()
			metadata_row[ 'size' ] = file_info_manager.size
			metadata_row[ 'mime' ] = HC.mime_string_lookup[ file_info_manager.mime ]
			metadata_row[ 'width' ] = file_info_manager.width
			metadata_row[ 'height' ] = file_info_manager.height
			metadata_row[ 'duration' ] = file_info_manager.duration
			metadata_row[ 'num_frames' ] = file_info_manager.num_frames
			metadata_row[ 'num_words' ] = file_info_manager.num_words

			tags_manager = media_result.GetTagsManager()

			service_names_to_statuses_to_tags = {}

			service_keys_to_statuses_to_tags = tags_manager.GetServiceKeysToStatusesToTags()

			for ( service_key, statuses_to_tags ) in service_keys_to_statuses_to_tags.items():
				if service_key not in service_keys_to_names:
					service_keys_to_names[ service_key ] = services_manager.GetName( service_key )

				service_name = service_keys_to_names[ service_key ]

				service_names_to_statuses_to_tags[ service_name ] = { str( status ) : list( tags ) for ( status, tags ) in statuses_to_tags.items() }

			metadata_row[ 'service_names_to_statuses_to_tags' ] = service_names_to_statuses_to_tags

			metadata.append( metadata_row )

		expected_metadata_result = { 'metadata' : metadata }

		HG.test_controller.SetRead( 'hash_ids_to_hashes', file_ids_to_hashes )
		HG.test_controller.SetRead( 'media_results', media_results )
		HG.test_controller.SetRead( 'media_results_from_ids', media_results )

		api_permissions.SetLastSearchResults( [ 1, 2, 3, 4, 5, 6 ] )

		# fail on non-permitted files

		path = '/get_files/file_metadata?file_ids={}&only_return_identifiers=true'.format( urllib.parse.quote( json.dumps( [ 1, 2, 3, 7 ] ) ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 403 )

		# fails on hashes even if the hashes are 'good'

		path = '/get_files/file_metadata?hashes={}&only_return_identifiers=true'.format( urllib.parse.quote( json.dumps( [ hash.hex() for hash in file_ids_to_hashes.values() ] ) ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 403 )

		# identifiers from file_ids

		path = '/get_files/file_metadata?file_ids={}&only_return_identifiers=true'.format( urllib.parse.quote( json.dumps( [ 1, 2, 3 ] ) ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		text = str( data, 'utf-8' )

		d = json.loads( text )

		self.assertEqual( d, expected_identifier_result )

		# metadata from file_ids

		path = '/get_files/file_metadata?file_ids={}'.format( urllib.parse.quote( json.dumps( [ 1, 2, 3 ] ) ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		text = str( data, 'utf-8' )

		d = json.loads( text )

		self.assertEqual( d, expected_metadata_result )

		# now from hashes

		api_permissions = set_up_permissions[ 'everything' ]

		access_key_hex = api_permissions.GetAccessKey().hex()

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex }

		# identifiers from hashes

		path = '/get_files/file_metadata?hashes={}&only_return_identifiers=true'.format( urllib.parse.quote( json.dumps( [ hash.hex() for hash in file_ids_to_hashes.values() ] ) ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		text = str( data, 'utf-8' )

		d = json.loads( text )

		self.assertEqual( d, expected_identifier_result )

		# metadata from hashes

		path = '/get_files/file_metadata?hashes={}'.format( urllib.parse.quote( json.dumps( [ hash.hex() for hash in file_ids_to_hashes.values() ] ) ) )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		text = str( data, 'utf-8' )

		d = json.loads( text )

		self.assertEqual( d, expected_metadata_result )

		# files and thumbs

		hash = b'\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3'
		hash_hex = hash.hex()

		size = 100
		mime = HC.IMAGE_PNG
		width = 20
		height = 20
		duration = None

		file_info_manager = Media.FileInfoManager( file_id, hash, size = size, mime = mime, width = width, height = height, duration = duration )

		service_keys_to_statuses_to_tags = { CC.LOCAL_TAG_SERVICE_KEY : { HC.CONTENT_STATUS_CURRENT : [ 'blue eyes', 'blonde hair' ], HC.CONTENT_STATUS_PENDING : [ 'bodysuit' ] } }

		tags_manager = Media.TagsManager( service_keys_to_statuses_to_tags )

		locations_manager = Media.LocationsManager( set(), set(), set(), set() )
		ratings_manager = Ratings.RatingsManager( {} )
		file_viewing_stats_manager = Media.FileViewingStatsManager( 0, 0, 0, 0 )

		media_result = Media.MediaResult( file_info_manager, tags_manager, locations_manager, ratings_manager, file_viewing_stats_manager )

		HG.test_controller.SetRead( 'media_results', ( media_result, ) )
		HG.test_controller.SetRead( 'media_results_from_ids', ( media_result, ) )

		path = os.path.join( HC.STATIC_DIR, 'hydrus.png' )

		file_path = HG.test_controller.client_files_manager.GetFilePath( hash, HC.IMAGE_PNG, check_file_exists = False )

		shutil.copy2( path, file_path )

		thumb_hash = b'\x17\xde\xd6\xee\x1b\xfa\x002\xbdj\xc0w\x92\xce5\xf0\x12~\xfe\x915\xb3\xb3tA\xac\x90F\x95\xc2T\xc5'

		path = os.path.join( HC.STATIC_DIR, 'hydrus_small.png' )

		thumb_path = HG.test_controller.client_files_manager._GenerateExpectedThumbnailPath( hash )

		shutil.copy2( path, thumb_path )

		api_permissions = set_up_permissions[ 'search_green_files' ]

		access_key_hex = api_permissions.GetAccessKey().hex()

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex }

		# let's fail first

		path = '/get_files/file?file_id={}'.format( 10 )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 403 )

		#

		path = '/get_files/thumbnail?file_id={}'.format( 10 )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 403 )

		#

		path = '/get_files/file?hash={}'.format( hash_hex )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 403 )

		#

		path = '/get_files/thumbnail?hash={}'.format( hash_hex )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 403 )

		# now succeed

		path = '/get_files/file?file_id={}'.format( 1 )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		self.assertEqual( hashlib.sha256( data ).digest(), hash )

		#

		path = '/get_files/thumbnail?file_id={}'.format( 1 )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		self.assertEqual( hashlib.sha256( data ).digest(), thumb_hash )

		#

		api_permissions = set_up_permissions[ 'everything' ]

		access_key_hex = api_permissions.GetAccessKey().hex()

		headers = { 'Hydrus-Client-API-Access-Key' : access_key_hex }

		#

		path = '/get_files/file?hash={}'.format( hash_hex )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		self.assertEqual( hashlib.sha256( data ).digest(), hash )

		#

		path = '/get_files/thumbnail?hash={}'.format( hash_hex )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 200 )

		self.assertEqual( hashlib.sha256( data ).digest(), thumb_hash )

		# now 404

		hash_404 = os.urandom( 32 )

		file_info_manager = Media.FileInfoManager( 123456, hash_404, size = size, mime = mime, width = width, height = height, duration = duration )

		media_result = Media.MediaResult( file_info_manager, tags_manager, locations_manager, ratings_manager, file_viewing_stats_manager )

		HG.test_controller.SetRead( 'media_results', ( media_result, ) )
		HG.test_controller.SetRead( 'media_results_from_ids', ( media_result, ) )

		#

		path = '/get_files/file?hash={}'.format( hash_404.hex() )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 404 )

		#

		path = '/get_files/thumbnail?hash={}'.format( hash_404.hex() )

		connection.request( 'GET', path, headers = headers )

		response = connection.getresponse()

		data = response.read()

		self.assertEqual( response.status, 404 )

		#

		os.unlink( file_path )
		os.unlink( thumb_path )

	def _test_permission_failures( self, connection, set_up_permissions ):
		pass

		# failed permission tests

	def test_client_api( self ):
		host = '127.0.0.1'
		port = 45869

		connection = http.client.HTTPConnection( host, port, timeout = 10 )

		self._test_basics( connection )
		set_up_permissions = self._test_client_api_basics( connection )
		self._test_add_files( connection, set_up_permissions )
		self._test_add_tags( connection, set_up_permissions )
		self._test_add_urls( connection, set_up_permissions )
		self._test_search_files( connection, set_up_permissions )
		self._test_permission_failures( connection, set_up_permissions )
		self._test_cors_fails( connection )

		connection.close()

		#

		port = 45899

		connection = http.client.HTTPConnection( host, port, timeout = 10 )

		self._test_cors_succeeds( connection )


code:
### IMPORTS ###
from lib.Client import Constants as CC, Networking, Caches, Services

from lib.ClientGUI import Management

from lib.Hydrus import Constants as HC, Data, Globals as HG

import collections, os, unittest

### CODE ###

class TestManagers( unittest.TestCase ):
	def test_services( self ):
		def test_service( service, key, service_type, name ):
			self.assertEqual( service.GetServiceKey(), key )
			self.assertEqual( service.GetServiceType(), service_type )
			self.assertEqual( service.GetName(), name )

		repo_key = Data.GenerateKey()
		repo_type = HC.TAG_REPOSITORY
		repo_name = 'test tag repo'

		repo = Services.GenerateService( repo_key, repo_type, repo_name )

		other_key = Data.GenerateKey()

		other = Services.GenerateService( other_key, HC.LOCAL_BOORU, 'booru' )

		services = []

		services.append( repo )
		services.append( other )

		HG.test_controller.SetRead( 'services', services )

		services_manager = Caches.ServicesManager( HG.client_controller )

		#

		service = services_manager.GetService( repo_key )

		test_service( service, repo_key, repo_type, repo_name )

		service = services_manager.GetService( other_key )

		#

		services = services_manager.GetServices( ( HC.TAG_REPOSITORY, ) )

		self.assertEqual( len( services ), 1 )

		self.assertEqual( services[0].GetServiceKey(), repo_key )

		#

		services = []

		services.append( repo )

		HG.test_controller.SetRead( 'services', services )

		services_manager.RefreshServices()

		self.assertRaises( Exception, services_manager.GetService, other_key )

	def test_undo( self ):
		hash_1 = Data.GenerateKey()
		hash_2 = Data.GenerateKey()
		hash_3 = Data.GenerateKey()

		command_1 = { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : [ Data.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ARCHIVE, { hash_1 } ) ] }
		command_2 = { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : [ Data.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_INBOX, { hash_2 } ) ] }
		command_3 = { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : [ Data.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ARCHIVE, { hash_1, hash_3 } ) ] }

		command_1_inverted = { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : [ Data.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_INBOX, { hash_1 } ) ] }
		command_2_inverted = { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : [ Data.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ARCHIVE, { hash_2 } ) ] }

		undo_manager = Caches.UndoManager( HG.client_controller )

		#

		HG.test_controller.ClearWrites( 'content_updates' )

		undo_manager.AddCommand( 'content_updates', command_1 )

		self.assertEqual( ( 'undo archive 1 files', None ), undo_manager.GetUndoRedoStrings() )

		undo_manager.AddCommand( 'content_updates', command_2 )

		self.assertEqual( ( 'undo inbox 1 files', None ), undo_manager.GetUndoRedoStrings() )

		undo_manager.Undo()

		self.assertEqual( ( 'undo archive 1 files', 'redo inbox 1 files' ), undo_manager.GetUndoRedoStrings() )

		self.assertEqual( HG.test_controller.GetWrite( 'content_updates' ), [ ( ( command_2_inverted, ), {} ) ] )

		undo_manager.Redo()

		self.assertEqual( HG.test_controller.GetWrite( 'content_updates' ), [ ( ( command_2, ), {} ) ] )

		self.assertEqual( ( 'undo inbox 1 files', None ), undo_manager.GetUndoRedoStrings() )

		undo_manager.Undo()

		self.assertEqual( HG.test_controller.GetWrite( 'content_updates' ), [ ( ( command_2_inverted, ), {} ) ] )

		undo_manager.Undo()

		self.assertEqual( HG.test_controller.GetWrite( 'content_updates' ), [ ( ( command_1_inverted, ), {} ) ] )

		self.assertEqual( ( None, 'redo archive 1 files' ), undo_manager.GetUndoRedoStrings() )

		undo_manager.AddCommand( 'content_updates', command_3 )

		self.assertEqual( ( 'undo archive 2 files', None ), undo_manager.GetUndoRedoStrings() )

license: wtfpl


code:
### IMPORT ###
from lib.Client import Daemons, Importing, ImportLocal, Paths, Constants as CC,

from lib.Hydrus import Constants as HC, Data, Globals as HG, Paths

import collections, os, shutil, stat, unittest

### CODE ###
with open( os.path.join( HC.STATIC_DIR, 'hydrus.png' ), 'rb' ) as f:
	EXAMPLE_FILE = f.read()

class TestDaemons( unittest.TestCase ):
	def test_import_folders_daemon( self ):
		test_dir = Paths.GetTempDir()

		try:
			HG.test_controller.SetRead( 'hash_status', ( CC.STATUS_UNKNOWN, None, '' ) )

			Paths.MakeSureDirectoryExists( test_dir )

			hydrus_png_path = os.path.join( HC.STATIC_DIR, 'hydrus.png' )

			Paths.MirrorFile( hydrus_png_path, os.path.join( test_dir, '0' ) )
			Paths.MirrorFile( hydrus_png_path, os.path.join( test_dir, '1' ) ) # previously imported
			Paths.MirrorFile( hydrus_png_path, os.path.join( test_dir, '2' ) )

			with open( os.path.join( test_dir, '3' ), 'wb' ) as f: f.write( b'blarg' ) # broken
			with open( os.path.join( test_dir, '4' ), 'wb' ) as f: f.write( b'blarg' ) # previously failed

			#

			actions = {}

			actions[ CC.STATUS_SUCCESSFUL_AND_NEW ] = CC.IMPORT_FOLDER_DELETE
			actions[ CC.STATUS_SUCCESSFUL_BUT_REDUNDANT ] = CC.IMPORT_FOLDER_DELETE
			actions[ CC.STATUS_DELETED ] = CC.IMPORT_FOLDER_DELETE
			actions[ CC.STATUS_ERROR ] = CC.IMPORT_FOLDER_IGNORE

			import_folder = ImportLocal.ImportFolder( 'imp', path = test_dir, actions = actions )

			HG.test_controller.SetRead( 'serialisable_names', [ 'imp' ] )
			HG.test_controller.SetRead( 'serialisable_named', import_folder )

			HG.test_controller.ClearWrites( 'import_file' )
			HG.test_controller.ClearWrites( 'serialisable' )

			Daemons.DAEMONCheckImportFolders()

			import_file = HG.test_controller.GetWrite( 'import_file' )

			self.assertEqual( len( import_file ), 3 )

			# I need to expand tests here with the new file system

			[ ( ( updated_import_folder, ), empty_dict ) ] = HG.test_controller.GetWrite( 'serialisable' )

			self.assertEqual( updated_import_folder, import_folder )

			self.assertTrue( not os.path.exists( os.path.join( test_dir, '0' ) ) )
			self.assertTrue( not os.path.exists( os.path.join( test_dir, '1' ) ) )
			self.assertTrue( not os.path.exists( os.path.join( test_dir, '2' ) ) )
			self.assertTrue( os.path.exists( os.path.join( test_dir, '3' ) ) )
			self.assertTrue( os.path.exists( os.path.join( test_dir, '4' ) ) )

		finally:
			shutil.rmtree( test_dir )


code:
### IMPORTS ###
from lib.Client import Constants as CC, ImportOptions, ImportFileSeeds

from lib.Hydrus import Constants as HC, Exceptions

import os, unittest


code:
### IMPORTS ###
from lib.Client import ImageHandling

from lib.Hydrus import Constants as HC

import collections, os, unittest

### CODE ###

class TestImageHandling( unittest.TestCase ):
	def test_phash( self ):
		phashes = ImageHandling.GenerateShapePerceptualHashes( os.path.join( HC.STATIC_DIR, 'hydrus.png' ), HC.IMAGE_PNG )

		self.assertEqual( phashes, set( [ b'\xb4M\xc7\xb2M\xcb8\x1c' ] ) )


code:
### IMPORTS ###
from lib.Client import Constants as CC, ImportFileSeeds, ImportOptions, Tags

from lib.Hydrus import Constants as HC, Data, Exceptions, Globals as HG

from mock import patch

import os, unittest

### CODE ###

class TestCheckerOptions( unittest.TestCase ):
	def test_checker_options( self ):
		regular_checker_options = ImportOptions.CheckerOptions( intended_files_per_check = 5, never_faster_than = 30, never_slower_than = 86400, death_file_velocity = ( 1, 86400 ) )
		fast_checker_options = ImportOptions.CheckerOptions( intended_files_per_check = 2, never_faster_than = 30, never_slower_than = 86400, death_file_velocity = ( 1, 86400 ) )
		slow_checker_options = ImportOptions.CheckerOptions( intended_files_per_check = 10, never_faster_than = 30, never_slower_than = 86400, death_file_velocity = ( 1, 86400 ) )
		callous_checker_options = ImportOptions.CheckerOptions( intended_files_per_check = 5, never_faster_than = 30, never_slower_than = 86400, death_file_velocity = ( 1, 60 ) )

		empty_file_seed_cache = ImportFileSeeds.FileSeedCache()

		file_seed_cache = ImportFileSeeds.FileSeedCache()

		last_check_time = 10000000

		one_day_before = last_check_time - 86400

		for i in range( 50 ):
			url = 'https://wew.lad/' + os.urandom( 16 ).hex()

			file_seed = ImportFileSeeds.FileSeed( ImportFileSeeds.FILE_SEED_TYPE_URL, url )

			file_seed.source_time = one_day_before - 10

			file_seed_cache.AddFileSeeds( ( file_seed, ) )

		for i in range( 50 ):
			url = 'https://wew.lad/' + os.urandom( 16 ).hex()

			file_seed = ImportFileSeeds.FileSeed( ImportFileSeeds.FILE_SEED_TYPE_URL, url )

			file_seed.source_time = last_check_time - 600

			file_seed_cache.AddFileSeeds( ( file_seed, ) )

		bare_file_seed_cache = ImportFileSeeds.FileSeedCache()

		url = 'https://wew.lad/' + 'early'

		file_seed = ImportFileSeeds.FileSeed( ImportFileSeeds.FILE_SEED_TYPE_URL, url )

		file_seed.source_time = one_day_before - 10

		bare_file_seed_cache.AddFileSeeds( ( file_seed, ) )

		url = 'https://wew.lad/' + 'in_time_delta'

		file_seed = ImportFileSeeds.FileSeed( ImportFileSeeds.FILE_SEED_TYPE_URL, url )

		file_seed.source_time = one_day_before + 10

		bare_file_seed_cache.AddFileSeeds( ( file_seed, ) )

		busy_file_seed_cache = ImportFileSeeds.FileSeedCache()

		url = 'https://wew.lad/' + 'early'

		file_seed = ImportFileSeeds.FileSeed( ImportFileSeeds.FILE_SEED_TYPE_URL, url )

		file_seed.source_time = one_day_before - 10

		busy_file_seed_cache.AddFileSeeds( ( file_seed, ) )

		for i in range( 8640 ):
			url = 'https://wew.lad/' + os.urandom( 16 ).hex()

			file_seed = ImportFileSeeds.FileSeed( ImportFileSeeds.FILE_SEED_TYPE_URL, url )

			file_seed.source_time = one_day_before + ( ( i + 1 ) * 10 ) - 1

			busy_file_seed_cache.AddFileSeeds( ( file_seed, ) )

		new_thread_file_seed_cache = ImportFileSeeds.FileSeedCache()

		for i in range( 10 ):
			url = 'https://wew.lad/' + os.urandom( 16 ).hex()

			file_seed = ImportFileSeeds.FileSeed( ImportFileSeeds.FILE_SEED_TYPE_URL, url )

			file_seed.source_time = last_check_time - 600

			new_thread_file_seed_cache.AddFileSeeds( ( file_seed, ) )

		# empty
		# should say ok if last_check_time is 0, so it can initialise
		# otherwise sperg out safely

		self.assertFalse( regular_checker_options.IsDead( empty_file_seed_cache, 0 ) )

		self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( empty_file_seed_cache, 0 ), 'no files yet' )

		self.assertEqual( regular_checker_options.GetNextCheckTime( empty_file_seed_cache, 0, 0 ), 0 )

		self.assertTrue( regular_checker_options.IsDead( empty_file_seed_cache, last_check_time ) )

		self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( empty_file_seed_cache, last_check_time ), 'no files, unable to determine velocity' )

		# regular
		# current velocity should be 50 files per day for the day ones and 0 files per min for the callous minute one

		self.assertFalse( regular_checker_options.IsDead( file_seed_cache, last_check_time ) )
		self.assertFalse( fast_checker_options.IsDead( file_seed_cache, last_check_time ) )
		self.assertFalse( slow_checker_options.IsDead( file_seed_cache, last_check_time ) )
		self.assertTrue( callous_checker_options.IsDead( file_seed_cache, last_check_time ) )

		self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( file_seed_cache, last_check_time ), 'at last check, found 50 files in previous 1 day' )
		self.assertEqual( fast_checker_options.GetPrettyCurrentVelocity( file_seed_cache, last_check_time ), 'at last check, found 50 files in previous 1 day' )
		self.assertEqual( slow_checker_options.GetPrettyCurrentVelocity( file_seed_cache, last_check_time ), 'at last check, found 50 files in previous 1 day' )
		self.assertEqual( callous_checker_options.GetPrettyCurrentVelocity( file_seed_cache, last_check_time ), 'at last check, found 0 files in previous 1 minute' )

		self.assertEqual( regular_checker_options.GetNextCheckTime( file_seed_cache, last_check_time, 0 ), last_check_time + 8640 )
		self.assertEqual( fast_checker_options.GetNextCheckTime( file_seed_cache, last_check_time, 0 ), last_check_time + 3456 )
		self.assertEqual( slow_checker_options.GetNextCheckTime( file_seed_cache, last_check_time, 0 ), last_check_time + 17280 )

		# bare
		# 1 files per day

		self.assertFalse( regular_checker_options.IsDead( bare_file_seed_cache, last_check_time ) )
		self.assertTrue( callous_checker_options.IsDead( bare_file_seed_cache, last_check_time ) )

		self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( bare_file_seed_cache, last_check_time ), 'at last check, found 1 files in previous 1 day' )

		self.assertEqual( regular_checker_options.GetNextCheckTime( bare_file_seed_cache, last_check_time, 0 ), last_check_time + 86400 )
		self.assertEqual( fast_checker_options.GetNextCheckTime( bare_file_seed_cache, last_check_time, 0 ), last_check_time + 86400 )
		self.assertEqual( slow_checker_options.GetNextCheckTime( bare_file_seed_cache, last_check_time, 0 ), last_check_time + 86400 )

		# busy
		# 8640 files per day, 6 files per minute

		self.assertFalse( regular_checker_options.IsDead( busy_file_seed_cache, last_check_time ) )
		self.assertFalse( fast_checker_options.IsDead( busy_file_seed_cache, last_check_time ) )
		self.assertFalse( slow_checker_options.IsDead( busy_file_seed_cache, last_check_time ) )
		self.assertFalse( callous_checker_options.IsDead( busy_file_seed_cache, last_check_time ) )

		self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( busy_file_seed_cache, last_check_time ), 'at last check, found 8,640 files in previous 1 day' )
		self.assertEqual( callous_checker_options.GetPrettyCurrentVelocity( busy_file_seed_cache, last_check_time ), 'at last check, found 6 files in previous 1 minute' )

		self.assertEqual( regular_checker_options.GetNextCheckTime( busy_file_seed_cache, last_check_time, 0 ), last_check_time + 50 )
		self.assertEqual( fast_checker_options.GetNextCheckTime( busy_file_seed_cache, last_check_time, 0 ), last_check_time + 30 )
		self.assertEqual( slow_checker_options.GetNextCheckTime( busy_file_seed_cache, last_check_time, 0 ), last_check_time + 100 )
		self.assertEqual( callous_checker_options.GetNextCheckTime( busy_file_seed_cache, last_check_time, 0 ), last_check_time + 50 )

		# new thread
		# only had files from ten mins ago, so timings are different

		self.assertFalse( regular_checker_options.IsDead( new_thread_file_seed_cache, last_check_time ) )
		self.assertFalse( fast_checker_options.IsDead( new_thread_file_seed_cache, last_check_time ) )
		self.assertFalse( slow_checker_options.IsDead( new_thread_file_seed_cache, last_check_time ) )
		self.assertTrue( callous_checker_options.IsDead( new_thread_file_seed_cache, last_check_time ) )

		self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( new_thread_file_seed_cache, last_check_time ), 'at last check, found 10 files in previous 10 minutes' )
		self.assertEqual( fast_checker_options.GetPrettyCurrentVelocity( new_thread_file_seed_cache, last_check_time ), 'at last check, found 10 files in previous 10 minutes' )
		self.assertEqual( slow_checker_options.GetPrettyCurrentVelocity( new_thread_file_seed_cache, last_check_time ), 'at last check, found 10 files in previous 10 minutes' )
		self.assertEqual( callous_checker_options.GetPrettyCurrentVelocity( new_thread_file_seed_cache, last_check_time ), 'at last check, found 0 files in previous 1 minute' )

		# these would be 360, 120, 600, but the 'don't check faster the time since last file post' bumps this up
		self.assertEqual( regular_checker_options.GetNextCheckTime( new_thread_file_seed_cache, last_check_time, 0 ), last_check_time + 600 )
		self.assertEqual( fast_checker_options.GetNextCheckTime( new_thread_file_seed_cache, last_check_time, 0 ), last_check_time + 600 )
		self.assertEqual( slow_checker_options.GetNextCheckTime( new_thread_file_seed_cache, last_check_time, 0 ), last_check_time + 600 )

		# Let's test these new static timings, where if faster_than == slower_than, we just add that period to the 'last_next_check_time' (e.g. checking every sunday night)

		static_checker_options = ImportOptions.CheckerOptions( intended_files_per_check = 5, never_faster_than = 3600, never_slower_than = 3600, death_file_velocity = ( 1, 3600 ) )

		self.assertTrue( static_checker_options.IsDead( bare_file_seed_cache, last_check_time ) )

		last_next_check_time = last_check_time - 200

		with patch.object( HydrusData, 'GetNow', return_value = last_check_time + 10 ):
			self.assertEqual( static_checker_options.GetNextCheckTime( new_thread_file_seed_cache, last_check_time, last_next_check_time ), last_next_check_time + 3600 )

class TestFileImportOptions( unittest.TestCase ):
	def test_file_import_options( self ):
		file_import_options = ImportOptions.FileImportOptions()

		exclude_deleted = False
		do_not_check_known_urls_before_importing = False
		do_not_check_hashes_before_importing = False
		allow_decompression_bombs = False
		min_size = None
		max_size = None
		max_gif_size = None
		min_resolution = None
		max_resolution = None

		file_import_options.SetPreImportOptions( exclude_deleted, do_not_check_known_urls_before_importing, do_not_check_hashes_before_importing, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution )

		automatic_archive = False
		associate_source_urls = False

		file_import_options.SetPostImportOptions( automatic_archive, associate_source_urls )

		present_new_files = True
		present_already_in_inbox_files = True
		present_already_in_archive_files = True

		file_import_options.SetPresentationOptions( present_new_files, present_already_in_inbox_files, present_already_in_archive_files )

		#

		self.assertFalse( file_import_options.ExcludesDeleted() )
		self.assertFalse( file_import_options.AllowsDecompressionBombs() )
		self.assertFalse( file_import_options.AutomaticallyArchives() )
		self.assertFalse( file_import_options.ShouldAssociateSourceURLs() )

		file_import_options.CheckFileIsValid( 65536, HC.IMAGE_JPEG, 640, 480 )
		file_import_options.CheckFileIsValid( 65536, HC.APPLICATION_7Z, None, None )

		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_AND_NEW, False ) )
		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_AND_NEW, True ) )
		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, False ) )
		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, True ) )

		self.assertFalse( file_import_options.ShouldPresent( CC.STATUS_DELETED, False ) )

		#

		exclude_deleted = True

		file_import_options.SetPreImportOptions( exclude_deleted, do_not_check_known_urls_before_importing, do_not_check_hashes_before_importing, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution )

		self.assertTrue( file_import_options.ExcludesDeleted() )
		self.assertFalse( file_import_options.AllowsDecompressionBombs() )
		self.assertFalse( file_import_options.AutomaticallyArchives() )

		#

		allow_decompression_bombs = True

		file_import_options.SetPreImportOptions( exclude_deleted, do_not_check_known_urls_before_importing, do_not_check_hashes_before_importing, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution )

		self.assertTrue( file_import_options.ExcludesDeleted() )
		self.assertTrue( file_import_options.AllowsDecompressionBombs() )
		self.assertFalse( file_import_options.AutomaticallyArchives() )

		#

		automatic_archive = True
		associate_source_urls  = True

		file_import_options.SetPostImportOptions( automatic_archive, associate_source_urls )

		self.assertTrue( file_import_options.ExcludesDeleted() )
		self.assertTrue( file_import_options.AllowsDecompressionBombs() )
		self.assertTrue( file_import_options.AutomaticallyArchives() )
		self.assertTrue( file_import_options.ShouldAssociateSourceURLs() )

		#

		min_size = 4096

		file_import_options.SetPreImportOptions( exclude_deleted, do_not_check_known_urls_before_importing, do_not_check_hashes_before_importing, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution )

		file_import_options.CheckFileIsValid( 65536, HC.IMAGE_JPEG, 640, 480 )

		with self.assertRaises( lib.Hydrus.Exceptions.SizeException ):
			file_import_options.CheckFileIsValid( 512, HC.IMAGE_JPEG, 640, 480 )

		#

		min_size = None
		max_size = 2000

		file_import_options.SetPreImportOptions( exclude_deleted, do_not_check_known_urls_before_importing, do_not_check_hashes_before_importing, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution )

		file_import_options.CheckFileIsValid( 1800, HC.IMAGE_JPEG, 640, 480 )

		with self.assertRaises( lib.Hydrus.Exceptions.SizeException ):
			file_import_options.CheckFileIsValid( 2200, HC.IMAGE_JPEG, 640, 480 )

		#

		max_size = None
		max_gif_size = 2000

		file_import_options.SetPreImportOptions( exclude_deleted, do_not_check_known_urls_before_importing, do_not_check_hashes_before_importing, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution )

		file_import_options.CheckFileIsValid( 1800, HC.IMAGE_JPEG, 640, 480 )
		file_import_options.CheckFileIsValid( 2200, HC.IMAGE_JPEG, 640, 480 )

		file_import_options.CheckFileIsValid( 1800, HC.IMAGE_GIF, 640, 480 )

		with self.assertRaises( lib.Hydrus.Exceptions.SizeException ):
			file_import_options.CheckFileIsValid( 2200, HC.IMAGE_GIF, 640, 480 )

		#

		max_gif_size = None
		min_resolution = ( 200, 100 )

		file_import_options.SetPreImportOptions( exclude_deleted, do_not_check_known_urls_before_importing, do_not_check_hashes_before_importing, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution )

		file_import_options.CheckFileIsValid( 65536, HC.IMAGE_JPEG, 640, 480 )

		with self.assertRaises( lib.Hydrus.Exceptions.SizeException ):
			file_import_options.CheckFileIsValid( 65536, HC.IMAGE_JPEG, 180, 480 )

		with self.assertRaises( lib.Hydrus.Exceptions.SizeException ):
			file_import_options.CheckFileIsValid( 65536, HC.IMAGE_JPEG, 640, 80 )

		file_import_options.CheckFileIsValid( 65536, HC.IMAGE_JPEG, 640, 180 )

		#

		min_resolution = None
		max_resolution = ( 3000, 4000 )

		file_import_options.SetPreImportOptions( exclude_deleted, do_not_check_known_urls_before_importing, do_not_check_hashes_before_importing, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution )

		file_import_options.CheckFileIsValid( 65536, HC.IMAGE_JPEG, 640, 480 )

		with self.assertRaises( lib.Hydrus.Exceptions.SizeException ):
			file_import_options.CheckFileIsValid( 65536, HC.IMAGE_JPEG, 3200, 480 )

		with self.assertRaises( lib.Hydrus.Exceptions.SizeException ):
			file_import_options.CheckFileIsValid( 65536, HC.IMAGE_JPEG, 640, 4200 )

		file_import_options.CheckFileIsValid( 65536, HC.IMAGE_JPEG, 2800, 3800 )

		#

		present_new_files = False

		file_import_options.SetPresentationOptions( present_new_files, present_already_in_inbox_files, present_already_in_archive_files )

		self.assertFalse( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_AND_NEW, False ) )
		self.assertFalse( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_AND_NEW, True ) )
		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, False ) )
		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, True ) )

		#

		present_new_files = True
		present_already_in_inbox_files = False

		file_import_options.SetPresentationOptions( present_new_files, present_already_in_inbox_files, present_already_in_archive_files )

		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_AND_NEW, False ) )
		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_AND_NEW, True ) )
		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, False ) )
		self.assertFalse( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, True ) )

		#

		present_already_in_inbox_files = True
		present_already_in_archive_files = False

		file_import_options.SetPresentationOptions( present_new_files, present_already_in_inbox_files, present_already_in_archive_files )

		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_AND_NEW, False ) )
		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_AND_NEW, True ) )
		self.assertFalse( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, False ) )
		self.assertTrue( file_import_options.ShouldPresent( CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, True ) )

class TestTagImportOptions( unittest.TestCase ):
	def test_basics( self ):
		some_tags = { 'bodysuit', 'character:samus aran', 'series:metroid' }
		example_hash = lib.Hydrus.Data.GenerateKey()

		#

		default_tag_import_options = ImportOptions.TagImportOptions()

		self.assertEqual( default_tag_import_options.ShouldFetchTagsEvenIfURLKnownAndFileAlreadyInDB(), False )
		self.assertEqual( default_tag_import_options.ShouldFetchTagsEvenIfHashKnownAndFileAlreadyInDB(), False )

		blacklist = default_tag_import_options.GetTagBlacklist()

		self.assertEqual( blacklist.Filter( some_tags ), some_tags )

		self.assertEqual( default_tag_import_options.GetServiceKeysToContentUpdates( CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags ), {} )

		#

		tag_import_options = ImportOptions.TagImportOptions( fetch_tags_even_if_url_recognised_and_file_already_in_db = True )

		self.assertEqual( tag_import_options.ShouldFetchTagsEvenIfURLKnownAndFileAlreadyInDB(), True )
		self.assertEqual( tag_import_options.ShouldFetchTagsEvenIfHashKnownAndFileAlreadyInDB(), False )

		#

		tag_import_options = ImportOptions.TagImportOptions( fetch_tags_even_if_hash_recognised_and_file_already_in_db = True )

		self.assertEqual( tag_import_options.ShouldFetchTagsEvenIfURLKnownAndFileAlreadyInDB(), False )
		self.assertEqual( tag_import_options.ShouldFetchTagsEvenIfHashKnownAndFileAlreadyInDB(), True )

	def test_filter( self ):
		some_tags = { 'bodysuit', 'character:samus aran', 'series:metroid' }
		example_hash = lib.Hydrus.Data.GenerateKey()
		example_service_key = lib.Hydrus.Data.GenerateKey()

		#

		tag_blacklist = Tags.TagFilter()

		tag_blacklist.SetRule( 'series:', CC.FILTER_BLACKLIST )

		service_keys_to_service_tag_import_options = { example_service_key : ImportOptions.ServiceTagImportOptions( get_tags = True ) }

		tag_import_options = ImportOptions.TagImportOptions( tag_blacklist = tag_blacklist, service_keys_to_service_tag_import_options = service_keys_to_service_tag_import_options )

		result = tag_import_options.GetServiceKeysToContentUpdates( CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags )

		self.assertIn( example_service_key, result )

		self.assertEqual( len( result ), 1 )

		content_updates = result[ example_service_key ]

		filtered_tags = { 'bodysuit', 'character:samus aran' }

		self.assertTrue( len( content_updates ), len( filtered_tags ) )

	def test_services( self ):
		some_tags = { 'bodysuit', 'character:samus aran', 'series:metroid' }
		example_hash = lib.Hydrus.Data.GenerateKey()
		example_service_key_1 = lib.Hydrus.Data.GenerateKey()
		example_service_key_2 = lib.Hydrus.Data.GenerateKey()

		#

		service_keys_to_service_tag_import_options = {}

		service_keys_to_service_tag_import_options[ example_service_key_1 ] = ImportOptions.ServiceTagImportOptions( get_tags = True )
		service_keys_to_service_tag_import_options[ example_service_key_2 ] = ImportOptions.ServiceTagImportOptions( get_tags = False )

		tag_import_options = ImportOptions.TagImportOptions( service_keys_to_service_tag_import_options = service_keys_to_service_tag_import_options )

		result = tag_import_options.GetServiceKeysToContentUpdates( CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags )

		self.assertIn( example_service_key_1, result )
		self.assertNotIn( example_service_key_2, result )

		self.assertTrue( len( result ), 2 )

		content_updates_1 = result[ example_service_key_1 ]

		self.assertEqual( len( content_updates_1 ), 3 )

class TestServiceTagImportOptions( unittest.TestCase ):
	def test_basics( self ):
		some_tags = { 'bodysuit', 'character:samus aran', 'series:metroid' }
		example_hash = lib.Hydrus.Data.GenerateKey()
		example_service_key = lib.Hydrus.Data.GenerateKey()

		#

		default_service_tag_import_options = ImportOptions.ServiceTagImportOptions()

		self.assertEqual( default_service_tag_import_options._get_tags, False )
		self.assertEqual( default_service_tag_import_options._additional_tags, [] )
		self.assertEqual( default_service_tag_import_options._to_new_files, True )
		self.assertEqual( default_service_tag_import_options._to_already_in_inbox, True )
		self.assertEqual( default_service_tag_import_options._to_already_in_archive, True )
		self.assertEqual( default_service_tag_import_options._only_add_existing_tags, False )

		self.assertEqual( default_service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags ), set() )

	def test_get_tags_filtering( self ):
		some_tags = { 'bodysuit', 'character:samus aran', 'series:metroid' }
		example_hash = lib.Hydrus.Data.GenerateKey()
		example_service_key = lib.Hydrus.Data.GenerateKey()

		#

		service_tag_import_options = ImportOptions.ServiceTagImportOptions( get_tags = True )

		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags ), some_tags )

		#

		only_namespaced = Tags.TagFilter()

		only_namespaced.SetRule( '', CC.FILTER_BLACKLIST )

		service_tag_import_options = ImportOptions.ServiceTagImportOptions( get_tags = True, get_tags_filter = only_namespaced )

		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags ), { 'character:samus aran', 'series:metroid' } )

		#

		only_samus = Tags.TagFilter()

		only_samus.SetRule( '', CC.FILTER_BLACKLIST )
		only_samus.SetRule( ':', CC.FILTER_BLACKLIST )
		only_samus.SetRule( 'character:samus aran', CC.FILTER_WHITELIST )

		service_tag_import_options = ImportOptions.ServiceTagImportOptions( get_tags = True, get_tags_filter = only_samus )

		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags ), { 'character:samus aran' } )

	def test_additional( self ):
		some_tags = { 'bodysuit', 'character:samus aran', 'series:metroid' }
		example_hash = lib.Hydrus.Data.GenerateKey()
		example_service_key = lib.Hydrus.Data.GenerateKey()

		#

		service_tag_import_options = ImportOptions.ServiceTagImportOptions( get_tags = True, additional_tags = [ 'wew' ] )

		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags ), some_tags.union( [ 'wew' ] ) )

	def test_application( self ):
		some_tags = { 'bodysuit', 'character:samus aran', 'series:metroid' }
		example_hash = lib.Hydrus.Data.GenerateKey()
		example_service_key = lib.Hydrus.Data.GenerateKey()

		#

		service_tag_import_options = ImportOptions.ServiceTagImportOptions( get_tags = True, to_new_files = True, to_already_in_inbox = False, to_already_in_archive = False )

		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags ), some_tags )
		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, True, example_hash, some_tags ), set() )
		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, False, example_hash, some_tags ), set() )

		#

		service_tag_import_options = ImportOptions.ServiceTagImportOptions( get_tags = True, to_new_files = False, to_already_in_inbox = True, to_already_in_archive = False )

		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags ), set() )
		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, True, example_hash, some_tags ), some_tags )
		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, False, example_hash, some_tags ), set() )

		#

		service_tag_import_options = ImportOptions.ServiceTagImportOptions( get_tags = True, to_new_files = False, to_already_in_inbox = False, to_already_in_archive = True )

		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags ), set() )
		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, True, example_hash, some_tags ), set() )
		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, False, example_hash, some_tags ), some_tags )

	def test_existing( self ):
		some_tags = { 'bodysuit', 'character:samus aran', 'series:metroid' }
		existing_tags = { 'character:samus aran', 'series:metroid' }
		example_hash = lib.Hydrus.Data.GenerateKey()
		example_service_key = lib.Hydrus.Data.GenerateKey()

		#

		HG.test_controller.SetRead( 'filter_existing_tags', existing_tags )

		service_tag_import_options = ImportOptions.ServiceTagImportOptions( get_tags = True, only_add_existing_tags = True )

		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags ), existing_tags )

		#

		some_tags = { 'explicit', 'bodysuit', 'character:samus aran', 'series:metroid' }
		existing_tags = { 'bodysuit' }

		only_unnamespaced = Tags.TagFilter()

		only_unnamespaced.SetRule( ':', CC.FILTER_BLACKLIST )

		HG.test_controller.SetRead( 'filter_existing_tags', existing_tags )

		service_tag_import_options = ImportOptions.ServiceTagImportOptions( get_tags = True, only_add_existing_tags = True, only_add_existing_tags_filter = only_unnamespaced )

		self.assertEqual( service_tag_import_options.GetTags( example_service_key, CC.STATUS_SUCCESSFUL_AND_NEW, True, example_hash, some_tags ), { 'bodysuit', 'character:samus aran', 'series:metroid' } )


code:
### IMPORTS ###
from lib.Client import Constants as CC, Defaults, ImportSubscriptions, Networking as ClientNetworking, NetworkingBandwidth, NetworkingDomain, NetworkingLogin, NetworkingSessions

from lib.Hydrus import Constants as HC, Data, Exceptions, Networking as HydrusNetworking, Globals as HG

from . import TestController

import os, collections, threading, time, unittest

from httmock import all_requests, urlmatch, HTTMock, response

from mock import patch

### STATICS ###
MISSING_RESPONSE = '404, bad result'

ERROR_RESPONSE = '500, it done broke'

EMPTY_RESPONSE = '''

	<title>results</title>

'''

### CODE ###
@all_requests

def catch_all( url, request ):
	raise Exception( 'An unexpected request for ' + url + ' came through in testing.' )

def get_good_response( urls_to_place ):
	response = '''

	<title>results</title>

  '''

	for url in urls_to_place:
		response += '''

	  '''

	response += '''
'''

	return response

class TestSubscription( unittest.TestCase ):
	def _PrepEngine( self ):
		mock_controller = TestController.MockController()
		bandwidth_manager = NetworkingBandwidth.NetworkBandwidthManager()
		session_manager = NetworkingSessions.NetworkSessionManager()
		domain_manager = NetworkingDomain.NetworkDomainManager()
		login_manager = NetworkingLogin.NetworkLoginManager()

		Defaults.SetDefaultDomainManagerData( domain_manager )

		engine = lib.Networking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )

		mock_controller.CallToThread( engine.MainLoop )

		return ( mock_controller, engine )

	def test_initial_sync( self ):
		# wait until I have searcher in here, so I can roll it all into domain_manager etc... rather than hitting db for gallery init and all that

		# use safebooru (i.e. gelb 0.2.0) for examples. pseudo html documents here that work with the parsers

		# 404 all file pages for now

		# refer to testclientnetworking for useful httmock examples

		# test:
		# initial is good to go with right stuff set up
		# a 404 gallery on initial
		# a 500 gallery on initial
		# a 200 gallery but empty result on initial
		# a 200 gallery typical good initial sync involving several pages
		# a subsequent 50 catch-up involving two pages
		# a user cancel on init
		# a user cancel on catch-up

		pass


code:
### IMPORTS ###
from lib.Client import Constants as CC, Defaults

from lib.ClientGUI import ListBoxes

from lib.Hydrus import Constants as HC, Globals as HG

from . import TestController

import os, collections, random, time, unittest, wx

### CODE ###

def DoClick( click, panel, do_delayed_ok_afterwards = False ):
	wx.QueueEvent( panel, click )

	if do_delayed_ok_afterwards:
		HG.test_controller.CallLaterWXSafe( panel, 1, PressKeyOnFocusedWindow, wx.WXK_RETURN )

	wx.YieldIfNeeded()

	time.sleep( 0.1 )

def GetAllClickableIndices( panel ):
	click = wx.MouseEvent( wx.wxEVT_LEFT_DOWN )

	current_y = 5

	click.SetX( 10 )
	click.SetY( current_y )

	all_clickable_indices = {}

	while panel._GetIndexUnderMouse( click ) is not None:
		index = panel._GetIndexUnderMouse( click )

		if index not in all_clickable_indices:
			all_clickable_indices[ index ] = current_y

		current_y += 5

		click.SetY( current_y )

	return all_clickable_indices

def PressKey( window, key ):
	window.SetFocus()

	uias = wx.UIActionSimulator()

	uias.Char( key )

def PressKeyOnFocusedWindow( key ):
	uias = wx.UIActionSimulator()

	uias.Char( key )

class TestListBoxes( unittest.TestCase ):
	def test_listbox_colour_options( self ):
		def wx_code():
			frame = TestController.TestFrame()

			try:
				initial_namespace_colours = { 'series' : ( 153, 101, 21 ), '' : ( 0, 111, 250 ), None : ( 114, 160, 193 ), 'creator' : ( 170, 0, 0 ) }

				panel = ListBoxes.ListBoxTagsColourOptions( frame, initial_namespace_colours )

				frame.SetPanel( panel )

				self.assertEqual( panel.GetNamespaceColours(), initial_namespace_colours )

				#

				new_namespace_colours = dict( initial_namespace_colours )
				new_namespace_colours[ 'character' ] = ( 0, 170, 0 )

				colour = wx.Colour( 0, 170, 0 )

				panel.SetNamespaceColour( 'character', colour )

				self.assertEqual( panel.GetNamespaceColours(), new_namespace_colours )

				#

				terms = set( panel._terms )
				ordered_terms = list( panel._ordered_terms )

				self.assertEqual( len( terms ), len( ordered_terms ) )

				#

				all_clickable_indices = GetAllClickableIndices( panel )

				self.assertEqual( len( list( all_clickable_indices.keys() ) ), len( terms ) )
				self.assertEqual( set( all_clickable_indices.keys() ), set( range( len( list( all_clickable_indices.keys() ) ) ) ) )

				#

				for ( index, y ) in list( all_clickable_indices.items() ):
					click = wx.MouseEvent( wx.wxEVT_LEFT_DOWN )

					click.SetX( 10 )
					click.SetY( y )

					DoClick( click, panel )

					self.assertEqual( panel.GetSelectedNamespaceColours(), dict( [ ordered_terms[ index ] ] ) )

				#

				current_y = 5

				click = wx.MouseEvent( wx.wxEVT_LEFT_DOWN )

				click.SetX( 10 )
				click.SetY( current_y )

				while panel._GetIndexUnderMouse( click ) is not None:
					current_y += 5

					click.SetY( current_y )

				DoClick( click, panel )

				self.assertEqual( panel.GetSelectedNamespaceColours(), {} )

				#

				if len( list( all_clickable_indices.keys() ) ) > 2:
					indices = random.sample( list( all_clickable_indices.keys() ), len( list(all_clickable_indices.keys() ) ) - 1 )

					for index in indices:
						click = wx.MouseEvent( wx.wxEVT_LEFT_DOWN )

						click.SetControlDown( True )

						click.SetX( 10 )
						click.SetY( all_clickable_indices[ index ] )

						DoClick( click, panel )

					expected_selected_terms = [ ordered_terms[ index ] for index in indices ]

					self.assertEqual( panel.GetSelectedNamespaceColours(), dict( expected_selected_terms ) )

				#

				random_index = random.choice( list( all_clickable_indices.keys() ) )

				while ordered_terms[ random_index ][0] in panel.PROTECTED_TERMS:
					random_index = random.choice( list( all_clickable_indices.keys() ) )

				del new_namespace_colours[ ordered_terms[ random_index ][0] ]

				# select nothing

				current_y = 5

				click = wx.MouseEvent( wx.wxEVT_LEFT_DOWN )

				click.SetX( 10 )
				click.SetY( current_y )

				while panel._GetIndexUnderMouse( click ) is not None:
					current_y += 5

					click.SetY( current_y )

				DoClick( click, panel )

				# select the random index

				click = wx.MouseEvent( wx.wxEVT_LEFT_DOWN )

				click.SetX( 10 )
				click.SetY( all_clickable_indices[ random_index ] )

				DoClick( click, panel )

				# now double-click to activate and hence remove

				doubleclick = wx.MouseEvent( wx.wxEVT_LEFT_DCLICK )

				doubleclick.SetX( 5 )
				doubleclick.SetY( all_clickable_indices[ random_index ] )

				DoClick( doubleclick, panel, do_delayed_ok_afterwards = True )

				self.assertEqual( panel.GetNamespaceColours(), new_namespace_colours )

			finally:
				frame.DestroyLater()

		HG.test_controller.CallBlockingToWX( HG.test_controller.win, wx_code )