This methods load the IWNLP.Lemmatizer json file and creates a dictionary
         of lowercased forms which maps each form to its possible lemmas.
329943
def load(self, lemmatizer_path):
        
        self.lemmatizer = {}
        with io.open(lemmatizer_path, encoding='utf-8') as data_file:
            raw = json.load(data_file)
            for entry in raw:
                self.lemmatizer[entry["Form"]] = entry["Lemmas"]
        self.apply_blacklist()
1247980
def mtable_to_dict(maf_id_file):
    

    out = []

    pattern = re.compile(r'(?P<id>\d+)\t\{(?P<json>.+)\}')
    for line in maf_id_file:
        m = pattern.match(line)
        text = '{"id": ' + m.group('id') + ', ' + m.group('json').replace("'", '"') + '}'
        out.append( json.loads(text) )

    return out
493356
def load_nicknames(self, file):
        
        with open(os.path.join(main_dir, file + '.dat'), 'r') as f:
            self.nicknames = json.load(f)
591638
private void createDictionaries(String file, String conllFormat) throws IOException
    {
        long start = System.currentTimeMillis();
        logger.debug("Creating dictionariesSet ... ");

        dictionariesSet.setCounters();

        DependencyReader reader = DependencyReader.createDependencyReader(conllFormat);

        reader.startReading(file);
        DependencyInstance dependencyInstance = reader.nextInstance();

        while (dependencyInstance != null) {
            //This loop sets values in dictionariesSet for later use
            dependencyInstance.setInstIds(dictionariesSet, coarseMap, conjWord);

            dependencyInstance = reader.nextInstance();
        }
        reader.close();

        dictionariesSet.closeCounters();

        synFactory.setTokenStart(dictionariesSet.lookupIndex(POS, "#TOKEN_START#"));
        synFactory.setTokenEnd(dictionariesSet.lookupIndex(POS, "#TOKEN_END#"));
        synFactory.setTokenMid(dictionariesSet.lookupIndex(POS, "#TOKEN_MID#"));

        dictionariesSet.stopGrowth(DEP_LABEL);
        dictionariesSet.stopGrowth(POS);
        dictionariesSet.stopGrowth(WORD);

        synFactory.setWordNumBits(Utils.log2((long) dictionariesSet.getDictionarySize(WORD) + 1));
        synFactory.setTagNumBits(Utils.log2((long) dictionariesSet.getDictionarySize(POS) + 1));
        synFactory.setDepNumBits(Utils.log2((long) dictionariesSet.getDictionarySize(DEP_LABEL) + 1));
        synFactory.setFlagBits(2*synFactory.getDepNumBits() + 4);

        types = new String[dictionariesSet.getDictionarySize(DEP_LABEL)];
        Dictionary labelDict = dictionariesSet.getDictionary(DEP_LABEL);
        Object[] keys = labelDict.toArray();
        for (Object key : keys) {
            int id = labelDict.lookupIndex(key);
            types[id - 1] = (String) key;
        }

        if(logger.isDebugEnabled()) {
            logger.debug(String.format("%d %d %n", NUM_WORD_FEAT_BITS, NUM_ARC_FEAT_BITS));
            logger.debug(String.format("Lexical items: %d (%d bits)%n",
                    dictionariesSet.getDictionarySize(WORD), synFactory.getWordNumBits()));
            logger.debug(String.format("Tag/label items: %d (%d bits)  %d (%d bits)%n",
                    dictionariesSet.getDictionarySize(POS), synFactory.getTagNumBits(),
                    dictionariesSet.getDictionarySize(DEP_LABEL), synFactory.getDepNumBits()));
            logger.debug(String.format("Flag Bits: %d%n", synFactory.getFlagBits()));
            logger.debug(String.format("Creation took [%d ms]%n", System.currentTimeMillis() - start));
        }

    }
1136761
def _lemmatise_assims(self, f, *args, **kwargs):
        
        forme_assimilee = self.assims(f)
        if forme_assimilee != f:
            for proposal in self._lemmatise(forme_assimilee, *args, **kwargs):
                yield proposal
140420
def from_json_file(cls, json_file):
        """"""
        with open(json_file, "r", encoding="utf-8") as reader:
            text = reader.read()
        return cls.from_dict(json.loads(text))
264835
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'models' in _dict:
            args['models'] = [
                TranslationModel._from_dict(x) for x in (_dict.get('models'))
            ]
        else:
            raise ValueError(
                'Required property \'models\' not present in TranslationModels JSON'
            )
        return cls(**args)
771834
def create_parser_options(lazy_mfcollection_parsing: bool = False) -> Dict[str, Dict[str, Any]]:
    
    return {MultifileCollectionParser.__name__: {'lazy_parsing': lazy_mfcollection_parsing}}
887184
def load_character_images(self, verbose=True):
        
        for dataset in self.character_sets:
            assert self.get_dataset(dataset) is True, "Datasets aren't properly downloaded, " \
                                                      "rerun to try again or download datasets manually."

        for dataset in self.character_sets:
            for image, label in self.load_dataset(dataset, verbose=verbose):
                yield image, label
20996
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'section_titles' in _dict:
            args['section_titles'] = [
                SectionTitles._from_dict(x)
                for x in (_dict.get('section_titles'))
            ]
        if 'leading_sentences' in _dict:
            args['leading_sentences'] = [
                LeadingSentence._from_dict(x)
                for x in (_dict.get('leading_sentences'))
            ]
        return cls(**args)
1263315
def load_json(filename, **kwargs):
    

    with open(filename, 'r', encoding='utf-8') as f:
        return json.load(f, **kwargs)
742880
private void loadAnnotateParameters() {
    this.annotateParser.addArgument("-m", "--model")
        .required(true)
        .help("It is required to provide a POS tagging model.");
    this.annotateParser.addArgument("-lm", "--lemmatizerModel")
         .required(true)
         .help("It is required to provide a lemmatizer model.");
    this.annotateParser.addArgument("-l", "--language")
        .choices("de", "en", "es", "eu", "fr", "gl", "it", "nl")
        .required(false)
        .help("Choose a language.");

    this.annotateParser.addArgument("--beamSize")
        .required(false)
        .setDefault(DEFAULT_BEAM_SIZE)
        .help("Choose beam size for decoding, it defaults to 3.");
    annotateParser.addArgument("-o", "--outputFormat")
        .required(false)
        .choices("naf", "conll")
        .setDefault(Flags.DEFAULT_OUTPUT_FORMAT)
        .help("Choose output format; it defaults to NAF.\n");
    this.annotateParser.addArgument("-mw", "--multiwords")
        .action(Arguments.storeTrue())
        .help("Use to detect and process multiwords.\n");
    this.annotateParser.addArgument("-d", "--dictag")
        .action(Arguments.storeTrue())
        .help("Post process POS tagger output with a monosemic dictionary.\n");
    this.annotateParser.addArgument("-a","--allMorphology")
        .action(Arguments.storeTrue())
        .help("Print all the POS tags and lemmas before disambiguation.\n");
  }
642380
def _load_vocab_file(vocab_file, reserved_tokens=None):
  """"""
  if reserved_tokens is None:
    reserved_tokens = RESERVED_TOKENS

  subtoken_list = []
  with tf.gfile.Open(vocab_file, mode="r") as f:
    for line in f:
      subtoken = _native_to_unicode(line.strip())
      subtoken = subtoken[1:-1]  # Remove surrounding single-quotes
      if subtoken in reserved_tokens:
        continue
      subtoken_list.append(_native_to_unicode(subtoken))
  return reserved_tokens + subtoken_list
94000
def _prepare_patterns(self, pattern_dict):
        """"""
        dict_compiled = {}
        dict_text = {}
        dict_dscr = {}
        for platform, patterns in pattern_dict.items():
            dict_text[platform] = {}
            dict_compiled[platform] = {}
            dict_dscr[platform] = {}

            for key, pattern in patterns.items():
                try:
                    text_pattern = None
                    compiled_pattern = None
                    description_pattern = None

                    if isinstance(pattern, str):
                        text_pattern = pattern
                        compiled_pattern = re.compile(text_pattern, re.MULTILINE)
                        description_pattern = key

                    elif isinstance(pattern, dict):
                        text_pattern = pattern['pattern']
                        compiled_pattern = re.compile(text_pattern, re.MULTILINE)
                        description_pattern = pattern['description']

                    elif isinstance(pattern, list):
                        text_pattern = self._concatenate_patterns(key, pattern)
                        compiled_pattern = re.compile(text_pattern, re.MULTILINE)
                        description_pattern = key

                    dict_text[platform][key] = text_pattern
                    dict_compiled[platform][key] = compiled_pattern
                    dict_dscr[platform][key] = description_pattern

                except re.error as e:
                    raise RuntimeError("Pattern compile error: {} ({}:{})".format(e.message, platform, key))

        return dict_compiled, dict_text, dict_dscr
131207
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'word' in _dict:
            args['word'] = _dict.get('word')
        else:
            raise ValueError(
                'Required property \'word\' not present in Word JSON')
        if 'sounds_like' in _dict:
            args['sounds_like'] = _dict.get('sounds_like')
        else:
            raise ValueError(
                'Required property \'sounds_like\' not present in Word JSON')
        if 'display_as' in _dict:
            args['display_as'] = _dict.get('display_as')
        else:
            raise ValueError(
                'Required property \'display_as\' not present in Word JSON')
        if 'count' in _dict:
            args['count'] = _dict.get('count')
        else:
            raise ValueError(
                'Required property \'count\' not present in Word JSON')
        if 'source' in _dict:
            args['source'] = _dict.get('source')
        else:
            raise ValueError(
                'Required property \'source\' not present in Word JSON')
        if 'error' in _dict:
            args['error'] = [
                WordError._from_dict(x) for x in (_dict.get('error'))
            ]
        return cls(**args)
1163716
def process_all_json_files(build_dir):
    """"""
    html_files = []
    for root, _, files in os.walk(build_dir):
        for filename in fnmatch.filter(files, '*.fjson'):
            if filename in ['search.fjson', 'genindex.fjson',
                            'py-modindex.fjson']:
                continue
            html_files.append(os.path.join(root, filename))
    page_list = []
    for filename in html_files:
        try:
            result = process_file(filename)
            if result:
                page_list.append(result)
        # we're unsure which exceptions can be raised
        except:  # noqa
            pass
    return page_list
303964
def _load_json_file(json_file):
        
        with io.open(json_file, encoding='utf-8') as data_file:
            try:
                json_content = json.load(data_file)
            except p_exception.JSONDecodeError:
                err_msg = u"JSONDecodeError: JSON file format error: {}".format(json_file)
                raise p_exception.FileFormatError(err_msg)

            FileUtils._check_format(json_file, json_content)
            return json_content