Upload 13 files

App_Function_Libraries/Personas/Character_Chat.py

@@ -0,0 +1,18 @@
+# Character_Chat.py
+# Description: Functions for character chat
+#
+# Imports
+#
+# External Imports
+#
+# Local Imports
+#
+# ############################################################################################################
+#
+# Functions:
+
+# FIXME - migrate functions from character_chat_tab to here
+
+#
+# End of Character_Chat.py
+############################################################################################################

App_Function_Libraries/Personas/cbs_handlers.py

@@ -0,0 +1,67 @@
+# cbs_handler.py
+import re
+import random
+from typing import List
+
+from App_Function_Libraries.Personas.models import CharacterCardV3
+
+
+class CBSHandler:
+    """Handles Curly Braced Syntaxes (CBS) in strings."""
+
+    CBS_PATTERN = re.compile(r'\{\{(.*?)\}\}')
+
+    def __init__(self, character_card: CharacterCardV3, user_display_name: str):
+        self.character_card = character_card
+        self.user_display_name = user_display_name
+
+    def replace_cbs(self, text: str) -> str:
+        """Replaces CBS in the given text with appropriate values."""
+        def replacer(match):
+            cbs_content = match.group(1).strip()
+            if cbs_content.lower() == 'char':
+                return self.character_card.data.nickname or self.character_card.data.name
+            elif cbs_content.lower() == 'user':
+                return self.user_display_name
+            elif cbs_content.lower().startswith('random:'):
+                options = self._split_escaped(cbs_content[7:])
+                return random.choice(options) if options else ''
+            elif cbs_content.lower().startswith('pick:'):
+                options = self._split_escaped(cbs_content[5:])
+                return random.choice(options) if options else ''
+            elif cbs_content.lower().startswith('roll:'):
+                return self._handle_roll(cbs_content[5:])
+            elif cbs_content.lower().startswith('//'):
+                return ''
+            elif cbs_content.lower().startswith('hidden_key:'):
+                # Placeholder for hidden_key logic
+                return ''
+            elif cbs_content.lower().startswith('comment:'):
+                # Placeholder for comment logic
+                return ''
+            elif cbs_content.lower().startswith('reverse:'):
+                return cbs_content[8:][::-1]
+            else:
+                # Unknown CBS; return as is or empty
+                return ''
+
+        return self.CBS_PATTERN.sub(replacer, text)
+
+    def _split_escaped(self, text: str) -> List[str]:
+        """Splits a string by commas, considering escaped commas."""
+        return [s.replace('\\,', ',') for s in re.split(r'(?<!\\),', text)]
+
+    def _handle_roll(self, value: str) -> str:
+        """Handles the roll:N CBS."""
+        value = value.lower()
+        if value.startswith('d'):
+            value = value[1:]
+        if value.isdigit():
+            return str(random.randint(1, int(value)))
+        return ''
+
+    def handle_comments(self, text: str) -> str:
+        """Handles comments in CBS."""
+        # Implementation depends on how comments should be displayed
+        # For simplicity, remove comments
+        return re.sub(r'\{\{comment:.*?\}\}', '', text)

App_Function_Libraries/Personas/ccv3_parser.py

@@ -0,0 +1,326 @@
+# ccv3_parser.py
+#
+#
+# Imports
+from typing import Any, Dict, List, Optional, Union
+import re
+#
+# External Imports
+#
+# Local Imports
+from App_Function_Libraries.Personas.models import Lorebook, Asset, CharacterCardV3, CharacterCardV3Data, Decorator, \
+    LorebookEntry
+from App_Function_Libraries.Personas.utils import validate_iso_639_1, extract_json_from_charx, parse_json_file, \
+    extract_text_chunks_from_png, decode_base64
+#
+############################################################################################################
+#
+# Functions:
+
+class CCv3ParserError(Exception):
+    """Custom exception for CCv3 Parser errors."""
+    pass
+
+
+class CharacterCardV3Parser:
+    REQUIRED_SPEC = 'chara_card_v3'
+    REQUIRED_VERSION = '3.0'
+
+    def __init__(self, input_data: Union[str, bytes], input_type: str):
+        """
+        Initialize the parser with input data.
+
+        :param input_data: The input data as a string or bytes.
+        :param input_type: The type of the input data: 'json', 'png', 'apng', 'charx'.
+        """
+        self.input_data = input_data
+        self.input_type = input_type.lower()
+        self.character_card: Optional[CharacterCardV3] = None
+
+    def parse(self):
+        """Main method to parse the input data based on its type."""
+        if self.input_type == 'json':
+            self.parse_json_input()
+        elif self.input_type in ['png', 'apng']:
+            self.parse_png_apng_input()
+        elif self.input_type == 'charx':
+            self.parse_charx_input()
+        else:
+            raise CCv3ParserError(f"Unsupported input type: {self.input_type}")
+
+    def parse_json_input(self):
+        """Parse JSON input directly."""
+        try:
+            data = parse_json_file(
+                self.input_data.encode('utf-8') if isinstance(self.input_data, str) else self.input_data)
+            self.character_card = self._build_character_card(data)
+        except Exception as e:
+            raise CCv3ParserError(f"Failed to parse JSON input: {e}")
+
+    def parse_png_apng_input(self):
+        """Parse PNG or APNG input by extracting 'ccv3' tEXt chunk."""
+        try:
+            text_chunks = extract_text_chunks_from_png(self.input_data)
+            if 'ccv3' not in text_chunks:
+                raise CCv3ParserError("PNG/APNG does not contain 'ccv3' tEXt chunk.")
+            ccv3_base64 = text_chunks['ccv3']
+            ccv3_json_bytes = decode_base64(ccv3_base64)
+            data = parse_json_file(ccv3_json_bytes)
+            self.character_card = self._build_character_card(data)
+        except Exception as e:
+            raise CCv3ParserError(f"Failed to parse PNG/APNG input: {e}")
+
+    def parse_charx_input(self):
+        """Parse CHARX input by extracting 'card.json' from the ZIP archive."""
+        try:
+            data = extract_json_from_charx(self.input_data)
+            self.character_card = self._build_character_card(data)
+        except Exception as e:
+            raise CCv3ParserError(f"Failed to parse CHARX input: {e}")
+
+    def _build_character_card(self, data: Dict[str, Any]) -> CharacterCardV3:
+        """Build the CharacterCardV3 object from parsed data."""
+        # Validate required fields
+        spec = data.get('spec')
+        spec_version = data.get('spec_version')
+        if spec != self.REQUIRED_SPEC:
+            raise CCv3ParserError(f"Invalid spec: Expected '{self.REQUIRED_SPEC}', got '{spec}'")
+        if spec_version != self.REQUIRED_VERSION:
+            # As per spec, should not reject but handle versions
+            # For now, proceed if version is >=3.0
+            try:
+                version_float = float(spec_version)
+                if version_float < 3.0:
+                    raise CCv3ParserError(f"Unsupported spec_version: '{spec_version}' (must be >= '3.0')")
+            except ValueError:
+                raise CCv3ParserError(f"Invalid spec_version format: '{spec_version}'")
+
+        data_field = data.get('data')
+        if not data_field:
+            raise CCv3ParserError("Missing 'data' field in CharacterCardV3 object.")
+
+        # Extract required fields
+        required_fields = ['name', 'description', 'tags', 'creator', 'character_version',
+                           'mes_example', 'extensions', 'system_prompt',
+                           'post_history_instructions', 'first_mes',
+                           'alternate_greetings', 'personality', 'scenario',
+                           'creator_notes', 'group_only_greetings']
+        for field_name in required_fields:
+            if field_name not in data_field:
+                raise CCv3ParserError(f"Missing required field in data: '{field_name}'")
+
+        # Parse assets
+        assets_data = data_field.get('assets', [{
+            'type': 'icon',
+            'uri': 'ccdefault:',
+            'name': 'main',
+            'ext': 'png'
+        }])
+        assets = self._parse_assets(assets_data)
+
+        # Parse creator_notes_multilingual
+        creator_notes_multilingual = data_field.get('creator_notes_multilingual')
+        if creator_notes_multilingual:
+            if not isinstance(creator_notes_multilingual, dict):
+                raise CCv3ParserError("'creator_notes_multilingual' must be a dictionary.")
+            # Validate ISO 639-1 codes
+            for lang_code in creator_notes_multilingual.keys():
+                if not validate_iso_639_1(lang_code):
+                    raise CCv3ParserError(f"Invalid language code in 'creator_notes_multilingual': '{lang_code}'")
+
+        # Parse character_book
+        character_book_data = data_field.get('character_book')
+        character_book = self._parse_lorebook(character_book_data) if character_book_data else None
+
+        # Build CharacterCardV3Data
+        character_card_data = CharacterCardV3Data(
+            name=data_field['name'],
+            description=data_field['description'],
+            tags=data_field['tags'],
+            creator=data_field['creator'],
+            character_version=data_field['character_version'],
+            mes_example=data_field['mes_example'],
+            extensions=data_field['extensions'],
+            system_prompt=data_field['system_prompt'],
+            post_history_instructions=data_field['post_history_instructions'],
+            first_mes=data_field['first_mes'],
+            alternate_greetings=data_field['alternate_greetings'],
+            personality=data_field['personality'],
+            scenario=data_field['scenario'],
+            creator_notes=data_field['creator_notes'],
+            character_book=character_book,
+            assets=assets,
+            nickname=data_field.get('nickname'),
+            creator_notes_multilingual=creator_notes_multilingual,
+            source=data_field.get('source'),
+            group_only_greetings=data_field['group_only_greetings'],
+            creation_date=data_field.get('creation_date'),
+            modification_date=data_field.get('modification_date')
+        )
+
+        return CharacterCardV3(
+            spec=spec,
+            spec_version=spec_version,
+            data=character_card_data
+        )
+
+    def _parse_assets(self, assets_data: List[Dict[str, Any]]) -> List[Asset]:
+        """Parse and validate assets."""
+        assets = []
+        for asset_data in assets_data:
+            # Validate required fields
+            for field in ['type', 'uri', 'ext']:
+                if field not in asset_data:
+                    raise CCv3ParserError(f"Asset missing required field: '{field}'")
+                if not isinstance(asset_data[field], str):
+                    raise CCv3ParserError(f"Asset field '{field}' must be a string.")
+            # Optional 'name'
+            name = asset_data.get('name', '')
+            # Validate 'ext'
+            ext = asset_data['ext'].lower()
+            if not re.match(r'^[a-z0-9]+$', ext):
+                raise CCv3ParserError(f"Invalid file extension in asset: '{ext}'")
+            # Append to assets list
+            assets.append(Asset(
+                type=asset_data['type'],
+                uri=asset_data['uri'],
+                name=name,
+                ext=ext
+            ))
+        return assets
+
+    def _parse_lorebook(self, lorebook_data: Dict[str, Any]) -> Lorebook:
+        """Parse and validate Lorebook object."""
+        # Validate Lorebook fields
+        if not isinstance(lorebook_data, dict):
+            raise CCv3ParserError("Lorebook must be a JSON object.")
+
+        # Extract fields with defaults
+        name = lorebook_data.get('name')
+        description = lorebook_data.get('description')
+        scan_depth = lorebook_data.get('scan_depth')
+        token_budget = lorebook_data.get('token_budget')
+        recursive_scanning = lorebook_data.get('recursive_scanning')
+        extensions = lorebook_data.get('extensions', {})
+        entries_data = lorebook_data.get('entries', [])
+
+        # Parse entries
+        entries = self._parse_lorebook_entries(entries_data)
+
+        return Lorebook(
+            name=name,
+            description=description,
+            scan_depth=scan_depth,
+            token_budget=token_budget,
+            recursive_scanning=recursive_scanning,
+            extensions=extensions,
+            entries=entries
+        )
+
+    def _parse_lorebook_entries(self, entries_data: List[Dict[str, Any]]) -> List[LorebookEntry]:
+        """Parse and validate Lorebook entries."""
+        entries = []
+        for entry_data in entries_data:
+            # Validate required fields
+            for field in ['keys', 'content', 'enabled', 'insertion_order']:
+                if field not in entry_data:
+                    raise CCv3ParserError(f"Lorebook entry missing required field: '{field}'")
+            if not isinstance(entry_data['keys'], list) or not all(isinstance(k, str) for k in entry_data['keys']):
+                raise CCv3ParserError("'keys' field in Lorebook entry must be a list of strings.")
+            if not isinstance(entry_data['content'], str):
+                raise CCv3ParserError("'content' field in Lorebook entry must be a string.")
+            if not isinstance(entry_data['enabled'], bool):
+                raise CCv3ParserError("'enabled' field in Lorebook entry must be a boolean.")
+            if not isinstance(entry_data['insertion_order'], (int, float)):
+                raise CCv3ParserError("'insertion_order' field in Lorebook entry must be a number.")
+
+            # Optional fields
+            use_regex = entry_data.get('use_regex', False)
+            constant = entry_data.get('constant')
+            selective = entry_data.get('selective')
+            secondary_keys = entry_data.get('secondary_keys')
+            position = entry_data.get('position')
+            name = entry_data.get('name')
+            priority = entry_data.get('priority')
+            entry_id = entry_data.get('id')
+            comment = entry_data.get('comment')
+
+            if selective and not isinstance(selective, bool):
+                raise CCv3ParserError("'selective' field in Lorebook entry must be a boolean.")
+            if secondary_keys:
+                if not isinstance(secondary_keys, list) or not all(isinstance(k, str) for k in secondary_keys):
+                    raise CCv3ParserError("'secondary_keys' field in Lorebook entry must be a list of strings.")
+            if position and not isinstance(position, str):
+                raise CCv3ParserError("'position' field in Lorebook entry must be a string.")
+
+            # Parse decorators from content
+            decorators = self._extract_decorators(entry_data['content'])
+
+            # Create LorebookEntry
+            entries.append(LorebookEntry(
+                keys=entry_data['keys'],
+                content=entry_data['content'],
+                enabled=entry_data['enabled'],
+                insertion_order=int(entry_data['insertion_order']),
+                use_regex=use_regex,
+                constant=constant,
+                selective=selective,
+                secondary_keys=secondary_keys,
+                position=position,
+                decorators=decorators,
+                name=name,
+                priority=priority,
+                id=entry_id,
+                comment=comment
+            ))
+        return entries
+
+    def _extract_decorators(self, content: str) -> List[Decorator]:
+        """Extract decorators from the content field."""
+        decorators = []
+        lines = content.splitlines()
+        for line in lines:
+            if line.startswith('@@'):
+                decorator = self._parse_decorator_line(line)
+                if decorator:
+                    decorators.append(decorator)
+        return decorators
+
+    def _parse_decorator_line(self, line: str) -> Optional[Decorator]:
+        """
+        Parses a single decorator line.
+
+        Example:
+            @@decorator_name value
+            @@@fallback_decorator value
+        """
+        fallback = None
+        if line.startswith('@@@'):
+            # Fallback decorator
+            name_value = line.lstrip('@').strip()
+            parts = name_value.split(' ', 1)
+            name = parts[0]
+            value = parts[1] if len(parts) > 1 else None
+            fallback = Decorator(name=name, value=value)
+            return fallback
+        elif line.startswith('@@'):
+            # Primary decorator
+            name_value = line.lstrip('@').strip()
+            parts = name_value.split(' ', 1)
+            name = parts[0]
+            value = parts[1] if len(parts) > 1 else None
+            # Check for fallback decorators in subsequent lines
+            # This assumes that fallback decorators follow immediately after the primary
+            # decorator in the content
+            # For simplicity, not implemented here. You can enhance this based on your needs.
+            return Decorator(name=name, value=value)
+        else:
+            return None
+
+    def get_character_card(self) -> Optional[CharacterCardV3]:
+        """Returns the parsed CharacterCardV3 object."""
+        return self.character_card
+
+#
+# End of ccv3_parser.py
+############################################################################################################

App_Function_Libraries/Personas/decorators.py

@@ -0,0 +1,48 @@
+# decorators.py
+from typing import List, Optional
+
+from App_Function_Libraries.Personas.models import Decorator
+
+
+# Assume Decorator class is already defined in models.py
+
+class DecoratorProcessor:
+    """Processes decorators for Lorebook entries."""
+
+    def __init__(self, decorators: List[Decorator]):
+        self.decorators = decorators
+
+    def process(self):
+        """Process decorators based on their definitions."""
+        for decorator in self.decorators:
+            # Implement processing logic based on decorator.name
+            if decorator.name == 'activate_only_after':
+                self._activate_only_after(decorator.value)
+            elif decorator.name == 'activate_only_every':
+                self._activate_only_every(decorator.value)
+            # Add more decorator handling as needed
+            else:
+                # Handle unknown decorators or ignore
+                pass
+
+    def _activate_only_after(self, value: Optional[str]):
+        """Handle @@activate_only_after decorator."""
+        if value and value.isdigit():
+            count = int(value)
+            # Implement logic to activate only after 'count' messages
+            pass
+        else:
+            # Invalid value; ignore or raise error
+            pass
+
+    def _activate_only_every(self, value: Optional[str]):
+        """Handle @@activate_only_every decorator."""
+        if value and value.isdigit():
+            frequency = int(value)
+            # Implement logic to activate every 'frequency' messages
+            pass
+        else:
+            # Invalid value; ignore or raise error
+            pass
+
+    # Implement other decorator handlers as needed

App_Function_Libraries/Personas/errors.py

@@ -0,0 +1,11 @@
+# errors.py
+# Description: Custom Exceptions for Personas
+#
+# Imports
+from typing import Any, Dict, List, Optional, Union
+#
+# Custom Exceptions
+
+class CCv3ParserError(Exception):
+    """Custom exception for CCv3 Parser errors."""
+    pass

App_Function_Libraries/Personas/models.py

@@ -0,0 +1,75 @@
+# models.py
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Union
+
+@dataclass
+class Asset:
+    type: str
+    uri: str
+    name: str = ""
+    ext: str = "unknown"
+
+@dataclass
+class Decorator:
+    name: str
+    value: Optional[str] = None
+    fallback: Optional['Decorator'] = None
+
+@dataclass
+class LorebookEntry:
+    keys: List[str]
+    content: str
+    enabled: bool
+    insertion_order: int
+    use_regex: bool = False
+    constant: Optional[bool] = None
+    selective: Optional[bool] = None
+    secondary_keys: Optional[List[str]] = None
+    position: Optional[str] = None
+    decorators: List[Decorator] = field(default_factory=list)
+    # Optional Fields
+    name: Optional[str] = None
+    priority: Optional[int] = None
+    id: Optional[Union[int, str]] = None
+    comment: Optional[str] = None
+
+@dataclass
+class Lorebook:
+    name: Optional[str] = None
+    description: Optional[str] = None
+    scan_depth: Optional[int] = None
+    token_budget: Optional[int] = None
+    recursive_scanning: Optional[bool] = None
+    extensions: Dict[str, Any] = field(default_factory=dict)
+    entries: List[LorebookEntry] = field(default_factory=list)
+
+@dataclass
+class CharacterCardV3Data:
+    name: str
+    description: str
+    tags: List[str]
+    creator: str
+    character_version: str
+    mes_example: str
+    extensions: Dict[str, Any]
+    system_prompt: str
+    post_history_instructions: str
+    first_mes: str
+    alternate_greetings: List[str]
+    personality: str
+    scenario: str
+    creator_notes: str
+    character_book: Optional[Lorebook] = None
+    assets: List[Asset] = field(default_factory=list)
+    nickname: Optional[str] = None
+    creator_notes_multilingual: Optional[Dict[str, str]] = None
+    source: Optional[List[str]] = None
+    group_only_greetings: List[str] = field(default_factory=list)
+    creation_date: Optional[int] = None
+    modification_date: Optional[int] = None
+
+@dataclass
+class CharacterCardV3:
+    spec: str
+    spec_version: str
+    data: CharacterCardV3Data

App_Function_Libraries/Personas/utils.py

@@ -0,0 +1,72 @@
+# utils.py
+import base64
+import json
+import re
+from typing import Any, Dict, List, Optional
+from zipfile import ZipFile, BadZipFile
+from io import BytesIO
+from PIL import Image, PngImagePlugin
+
+
+def decode_base64(data: str) -> bytes:
+    """Decodes a Base64 encoded string."""
+    try:
+        return base64.b64decode(data)
+    except base64.binascii.Error as e:
+        raise ValueError(f"Invalid Base64 data: {e}")
+
+
+def extract_text_chunks_from_png(png_bytes: bytes) -> Dict[str, str]:
+    """Extracts tEXt chunks from a PNG/APNG file."""
+    try:
+        with Image.open(BytesIO(png_bytes)) as img:
+            info = img.info
+            return info
+    except Exception as e:
+        raise ValueError(f"Failed to extract text chunks: {e}")
+
+
+def extract_json_from_charx(charx_bytes: bytes) -> Dict[str, Any]:
+    """Extracts and parses card.json from a CHARX file."""
+    try:
+        with ZipFile(BytesIO(charx_bytes)) as zip_file:
+            if 'card.json' not in zip_file.namelist():
+                raise ValueError("CHARX file does not contain card.json")
+            with zip_file.open('card.json') as json_file:
+                return json.load(json_file)
+    except BadZipFile:
+        raise ValueError("Invalid CHARX file: Not a valid zip archive")
+    except Exception as e:
+        raise ValueError(f"Failed to extract JSON from CHARX: {e}")
+
+
+def parse_json_file(json_bytes: bytes) -> Dict[str, Any]:
+    """Parses a JSON byte stream."""
+    try:
+        return json.loads(json_bytes.decode('utf-8'))
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid JSON data: {e}")
+
+
+def validate_iso_639_1(code: str) -> bool:
+    """Validates if the code is a valid ISO 639-1 language code."""
+    # For brevity, a small subset of ISO 639-1 codes
+    valid_codes = {
+        'en', 'es', 'fr', 'de', 'it', 'pt', 'ru', 'zh', 'ja', 'ko',
+        # Add more as needed
+    }
+    return code in valid_codes
+
+
+def parse_uri(uri: str) -> Dict[str, Any]:
+    """Parses the URI field and categorizes its type."""
+    if uri.startswith('http://') or uri.startswith('https://'):
+        return {'scheme': 'http', 'value': uri}
+    elif uri.startswith('embeded://'):
+        return {'scheme': 'embeded', 'value': uri.replace('embeded://', '')}
+    elif uri.startswith('ccdefault:'):
+        return {'scheme': 'ccdefault', 'value': None}
+    elif uri.startswith('data:'):
+        return {'scheme': 'data', 'value': uri}
+    else:
+        return {'scheme': 'unknown', 'value': uri}