Extracts sentence-level transcriptions, translations and wavs from the
    Na Pangloss XML and WAV files. But otherwise doesn't preprocess them.
421055
def trim_wavs(org_wav_dir=ORG_WAV_DIR,
              tgt_wav_dir=TGT_WAV_DIR,
              org_xml_dir=ORG_XML_DIR):
    

    logging.info("Trimming wavs...")

    if not os.path.exists(os.path.join(tgt_wav_dir, "TEXT")):
        os.makedirs(os.path.join(tgt_wav_dir, "TEXT"))
    if not os.path.exists(os.path.join(tgt_wav_dir, "WORDLIST")):
        os.makedirs(os.path.join(tgt_wav_dir, "WORDLIST"))

    for fn in os.listdir(org_xml_dir):
        path = os.path.join(org_xml_dir, fn)
        prefix, _ = os.path.splitext(fn)

        if os.path.isdir(path):
            continue
        if not path.endswith(".xml"):
            continue

        logging.info("Trimming wavs from {}".format(fn))

        rec_type, _, times, _ = pangloss.get_sents_times_and_translations(path)

        # Extract the wavs given the times.
        for i, (start_time, end_time) in enumerate(times):
            if prefix.endswith("PLUSEGG"):
                in_wav_path = os.path.join(org_wav_dir, prefix.upper()[:-len("PLUSEGG")]) + ".wav"
            else:
                in_wav_path = os.path.join(org_wav_dir, prefix.upper()) + ".wav"
            headmic_path = os.path.join(org_wav_dir, prefix.upper()) + "_HEADMIC.wav"
            if os.path.isfile(headmic_path):
                in_wav_path = headmic_path

            out_wav_path = os.path.join(tgt_wav_dir, rec_type, "%s.%d.wav" % (prefix, i))
            if not os.path.isfile(in_wav_path):
                raise PersephoneException("{} not a file.".format(in_wav_path))
            start_time = start_time * ureg.seconds
            end_time = end_time * ureg.seconds
            wav.trim_wav_ms(Path(in_wav_path), Path(out_wav_path),
                            start_time.to(ureg.milliseconds).magnitude,
                            end_time.to(ureg.milliseconds).magnitude)
397314
def transcribe(files=[], pre=10, post=50):
    ''''''

    total = len(files)

    for i, f in enumerate(files):
        filename = f.replace('.temp.wav', '') + '.transcription.txt'

        if os.path.exists(filename) is False:
            print(str(i+1) + '/' + str(total) + ' Transcribing ' + f)
            transcript = subprocess.check_output(['pocketsphinx_continuous', '-infile', f, '-time', 'yes', '-logfn', '/dev/null', '-vad_prespeech', str(pre), '-vad_postspeech', str(post)])

            with open(filename, 'w') as outfile:
                outfile.write(transcript.decode('utf8'))

            os.remove(f)
1323956
def preprocess_files(self, prefix):
            
            if prefix is None: return
            files = ("bin/yang2dsdl", "man/man1/yang2dsdl.1",
                     "pyang/plugins/jsonxsl.py")
            regex = re.compile("^(.*)/usr/local(.*)$")
            for f in files:
                  inf = open(f)
                  cnt = inf.readlines()
                  inf.close()
                  ouf = open(f,"w")
                  for line in cnt:
                        mo = regex.search(line)
                        if mo is None:
                              ouf.write(line)
                        else:
                              ouf.write(mo.group(1) + prefix + mo.group(2) +
                                        "\n")
                  ouf.close()
1303663
def extract_keys_from_files(localizable_files)
      keys_from_file = {}
      localizable_files.each do |file|
        lines = File.readlines(file)

        # Grab just the keys, we don't need the translation
        keys = lines.map { |e| e.split("=").first }
        # Filter newlines and comments
        keys = keys.select do |e|
          e != "\n" && !e.start_with?("/*") && !e.start_with?("//")
        end
        keys_from_file[file] = keys
      end
      keys_from_file
    end
939722
def from_tokens(cls, path:PathOrStr, trn_tok:Collection[Collection[str]], trn_lbls:Collection[Union[int,float]],
                 val_tok:Collection[Collection[str]], val_lbls:Collection[Union[int,float]], vocab:Vocab=None,
                 tst_tok:Collection[Collection[str]]=None, classes:Collection[Any]=None, max_vocab:int=60000, min_freq:int=3,
                 **kwargs) -> DataBunch:
        ""
        processor = NumericalizeProcessor(vocab=vocab, max_vocab=max_vocab, min_freq=min_freq)
        src = ItemLists(path, TextList(trn_tok, path=path, processor=processor),
                        TextList(val_tok, path=path, processor=processor))
        src = src.label_for_lm() if cls==TextLMDataBunch else src.label_from_lists(trn_lbls, val_lbls, classes=classes)
        if tst_tok is not None: src.add_test(TextList(tst_tok, path=path))
        return src.databunch(**kwargs)
942640
def fetch_all_treebanks(ud_dir, languages, corpus, best_per_language):
    "
    all_treebanks = dict()
    treebank_size = dict()
    for l in languages:
        all_treebanks[l] = []
        treebank_size[l] = 0

    for treebank_dir in ud_dir.iterdir():
        if treebank_dir.is_dir():
            for txt_path in treebank_dir.iterdir():
                if txt_path.name.endswith('-ud-' + corpus + '.txt'):
                    file_lang = txt_path.name.split('_')[0]
                    if file_lang in languages:
                        gold_path = treebank_dir / txt_path.name.replace('.txt', '.conllu')
                        stats_xml = treebank_dir / "stats.xml"
                        # ignore treebanks where the texts are not publicly available
                        if not _contains_blinded_text(stats_xml):
                            if not best_per_language:
                                all_treebanks[file_lang].append(txt_path)
                            # check the tokens in the gold annotation to keep only the biggest treebank per language
                            else:
                                with gold_path.open(mode='r', encoding='utf-8') as gold_file:
                                    gold_ud = conll17_ud_eval.load_conllu(gold_file)
                                    gold_tokens = len(gold_ud.tokens)
                                if treebank_size[file_lang] < gold_tokens:
                                    all_treebanks[file_lang] = [txt_path]
                                    treebank_size[file_lang] = gold_tokens

    return all_treebanks
949851
def main(out_path, ud_dir, check_parse=False, langs=ALL_LANGUAGES, exclude_trained_models=False, exclude_multi=False,
         hide_freq=False, corpus='train', best_per_language=False):
    "
    languages = [lang.strip() for lang in langs.split(",")]

    print_freq_tasks = []
    if not hide_freq:
        print_freq_tasks = ['Tokens']

    # fetching all relevant treebank from the directory
    treebanks = fetch_all_treebanks(ud_dir, languages, corpus, best_per_language)

    print()
    print("Loading all relevant models for", languages)
    models = dict()

    # multi-lang model
    multi = None
    if not exclude_multi and not check_parse:
        multi = load_model('xx_ent_wiki_sm', add_sentencizer=True)

    # initialize all models with the multi-lang model
    for lang in languages:
        models[lang] = [multi] if multi else []
        # add default models if we don't want to evaluate parsing info
        if not check_parse:
            # Norwegian is 'nb' in spaCy but 'no' in the UD corpora
            if lang == 'no':
                models['no'].append(load_default_model_sentencizer('nb'))
            else:
                models[lang].append(load_default_model_sentencizer(lang))

    # language-specific trained models
    if not exclude_trained_models:
        if 'de' in models:
            models['de'].append(load_model('de_core_news_sm'))
        if 'es' in models:
            models['es'].append(load_model('es_core_news_sm'))
            models['es'].append(load_model('es_core_news_md'))
        if 'pt' in models:
            models['pt'].append(load_model('pt_core_news_sm'))
        if 'it' in models:
            models['it'].append(load_model('it_core_news_sm'))
        if 'nl' in models:
            models['nl'].append(load_model('nl_core_news_sm'))
        if 'en' in models:
            models['en'].append(load_model('en_core_web_sm'))
            models['en'].append(load_model('en_core_web_md'))
            models['en'].append(load_model('en_core_web_lg'))
        if 'fr' in models:
            models['fr'].append(load_model('fr_core_news_sm'))
            models['fr'].append(load_model('fr_core_news_md'))

    with out_path.open(mode='w', encoding='utf-8') as out_file:
        run_all_evals(models, treebanks, out_file, check_parse, print_freq_tasks)
347685
def canonical_transcripts(gtf, out_file):
    
    if file_exists(out_file):
        return out_file
    db = get_gtf_db(gtf)
    with file_transaction(out_file) as tx_out_file:
        with open(tx_out_file, "w") as out_handle:
            for gene in db.features_of_type('gene'):
                exon_list = []
                for ti, transcript in enumerate(db.children(gene, level=1)):
                    cds_len = 0
                    total_len = 0
                    exons = list(db.children(transcript, level=1))
                    for exon in exons:
                        exon_length = len(exon)
                        if exon.featuretype == 'CDS':
                            cds_len += exon_length
                        total_len += exon_length

                    exon_list.append((cds_len, total_len, transcript, exons))

                # If we have CDS, then use the longest coding transcript
                if max(i[0] for i in exon_list) > 0:
                    best = sorted(exon_list)[0]
                # Otherwise, just choose the longest
                else:
                    best = sorted(exon_list, key=lambda x: x[1])[0]
                for exon in db.children(best[2], level=1):
                    out_handle.write(str(exon) + "\n")
    return out_file
930501
function extractRegions(peaks, duration) {
    // Silence params
    var minValue = 0.0015;
    var minSeconds = 0.25;

    var length = peaks.length;
    var coef = duration / length;
    var minLen = minSeconds / coef;

    // Gather silence indeces
    var silences = [];
    Array.prototype.forEach.call(peaks, function(val, index) {
        if (Math.abs(val) <= minValue) {
            silences.push(index);
        }
    });

    // Cluster silence values
    var clusters = [];
    silences.forEach(function(val, index) {
        if (clusters.length && val == silences[index - 1] + 1) {
            clusters[clusters.length - 1].push(val);
        } else {
            clusters.push([val]);
        }
    });

    // Filter silence clusters by minimum length
    var fClusters = clusters.filter(function(cluster) {
        return cluster.length >= minLen;
    });

    // Create regions on the edges of silences
    var regions = fClusters.map(function(cluster, index) {
        var next = fClusters[index + 1];
        return {
            start: cluster[cluster.length - 1],
            end: next ? next[0] : length - 1
        };
    });

    // Add an initial region if the audio doesn't start with silence
    var firstCluster = fClusters[0];
    if (firstCluster && firstCluster[0] != 0) {
        regions.unshift({
            start: 0,
            end: firstCluster[firstCluster.length - 1]
        });
    }

    // Filter regions by minimum length
    var fRegions = regions.filter(function(reg) {
        return reg.end - reg.start >= minLen;
    });

    // Return time-based regions
    return fRegions.map(function(reg) {
        return {
            start: Math.round(reg.start * coef * 10) / 10,
            end: Math.round(reg.end * coef * 10) / 10
        };
    });
}
615629
def compose_from_transcript(files, search, searchtype):
    """"""

    final_segments = []

    if searchtype in ['re', 'word', 'franken', 'fragment']:
        if searchtype == 're':
            searchtype = 'sentence'

        segments = audiogrep.search(search, files, mode=searchtype, regex=True)
        for seg in segments:
            seg['file'] = seg['file'].replace('.transcription.txt', '')
            seg['line'] = seg['words']
            final_segments.append(seg)

    elif searchtype in ['hyper', 'pos']:
        for s in audiogrep.convert_timestamps(files):
            for w in s['words']:
                if search_line(w[0], search, searchtype):
                    seg = {
                        'file': s['file'].replace('.transcription.txt',''),
                        'line': w[0],
                        'start': float(w[1]),
                        'end': float(w[2])
                    }
                    final_segments.append(seg)

    return final_segments
339145
def clean():
    """"""
    d = ['build', 'dist', 'scikits.audiolab.egg-info', HTML_DESTDIR,
            PDF_DESTDIR]
    for i in d:
        paver.path.path(i).rmtree()

    (paver.path.path('docs') / options.sphinx.builddir).rmtree()
619061
def clean_translated_locales(configuration, langs=None):
    
    if not langs:
        langs = configuration.translated_locales
    for locale in langs:
        clean_locale(configuration, locale)
1305603
def remove_namespaces(self, rpc_reply):
        """"""
        self.__xslt=self.__transform_reply
        self.__parser = etree.XMLParser(remove_blank_text=True)
        self.__xslt_doc = etree.parse(io.BytesIO(self.__xslt), self.__parser)
        self.__transform = etree.XSLT(self.__xslt_doc)
        self.__root = etree.fromstring(str(self.__transform(etree.parse(StringIO(str(rpc_reply))))))
        return self.__root
68997
def _walk_librispeech_dir(directory):
  """"""
  directory = os.path.join(directory, "LibriSpeech")
  for path, _, files in tf.io.gfile.walk(directory):
    if not files:
      continue

    transcript_file = [f for f in files if f.endswith(".txt")]
    if not transcript_file:
      continue
    assert len(transcript_file) == 1
    transcript_file, = transcript_file
    transcripts = {}
    with tf.io.gfile.GFile(os.path.join(path, transcript_file)) as f:
      for line in f:
        line = line.strip()
        key, transcript = line.split(" ", 1)
        transcripts[key] = transcript
    audio_files = [f for f in files if not f.endswith(".txt")]
    for audio_file in audio_files:
      assert audio_file.endswith(".flac")
      key = audio_file[:-len(".flac")]
      transcript = transcripts[key]
      speaker_id, chapter_id = [int(el) for el in key.split("-")[:2]]
      yield LibrispeechExample(
          speaker_id=speaker_id,
          chapter_id=chapter_id,
          audio_file=os.path.join(path, audio_file),
          transcript=transcript)
244029
def get_example_extractions(fname):
    ""
    with open(fname, 'r') as f:
        sentences = f.read().splitlines()
    rdf_xml_dict = {}
    for sentence in sentences:
        logger.info("Reading \"%s\"..." % sentence)
        html = tc.send_query(sentence, 'cwms')
        try:
            rdf_xml_dict[sentence] = tc.get_xml(html, 'rdf:RDF',
                                                fail_if_empty=True)
        except AssertionError as e:
            logger.error("Got error for %s." % sentence)
            logger.exception(e)
    return rdf_xml_dict
256805
def DoxyfileParse(file_contents):
   
   data = {}

   import shlex
   lex = shlex.shlex(instream = file_contents, posix = True)
   lex.wordchars += "*+./-:"
   lex.whitespace = lex.whitespace.replace("\n", "")
   lex.escape = ""

   lineno = lex.lineno
   token = lex.get_token()
   key = token   # the first token should be a key
   last_token = ""
   key_token = False
   next_key = False
   new_data = True

   def append_data(data, key, new_data, token):
      if new_data or len(data[key]) == 0:
         data[key].append(token)
      else:
         data[key][-1] += token

   while token:
      if token in ['\n']:
         if last_token not in ['\\']:
            key_token = True
      elif token in ['\\']:
         pass
      elif key_token:
         key = token
         key_token = False
      else:
         if token == "+=":
            if not data.has_key(key):
               data[key] = list()
         elif token == "=":
            if key == "TAGFILES" and data.has_key(key):
               append_data( data, key, False, "=" )
               new_data=False
            else:
               data[key] = list()
         else:
            append_data( data, key, new_data, token )
            new_data = True

      last_token = token
      token = lex.get_token()

      if last_token == '\\' and token != '\n':
         new_data = False
         append_data( data, key, new_data, '\\' )

   # compress lists of len 1 into single strings
   for (k, v) in data.items():
      if len(v) == 0:
         data.pop(k)

      # items in the following list will be kept as lists and not converted to strings
      if k in ["INPUT", "FILE_PATTERNS", "EXCLUDE_PATTERNS", "TAGFILES"]:
         continue

      if len(v) == 1:
         data[k] = v[0]

   return data
779327
public void parseFile(File file, boolean utterancePerDoc) {
        try {
            // Build an xml document.
            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
            DocumentBuilder db = dbf.newDocumentBuilder();
            Document doc = db.parse(file);

            // Extract all utterances.
            NodeList utterances = doc.getElementsByTagName("u");
            StringBuilder fileBuilder = new StringBuilder();
            for (int i = 0; i < utterances.getLength(); ++i) {
                // Extract all words from the utterance
                Element item = (Element) utterances.item(i);
                NodeList words = item.getElementsByTagName("w");
                StringBuilder utteranceBuilder = new StringBuilder();

                // Iterate over the words and get just the word text.
                List<String> wordsInUtterance =
                    new ArrayList<String>(words.getLength());
                for (int j = 0; j < words.getLength(); ++j) {
                    // Get the part of speech tag.
                    Element wordNode = (Element) words.item(j);
                    NodeList posNodeList = wordNode.getElementsByTagName("pos");
                    String word = wordNode.getFirstChild().getNodeValue();
                    if (posNodeList.getLength() > 0) {
                        Node posNode = 
                            posNodeList.item(0).getFirstChild().getFirstChild();
                        String pos = posNode.getNodeValue();
                        posTags.put(word, pos);
                        if (appendPosTags)
                            word += "-" + pos;
                    }
                    wordsInUtterance.add(word);
                }

                // Each of the <a> nodes contains additional information about
                // the currnet utterances.  This may be syntactic information,
                // comments on the scene, descriptions of the action, or
                // clarification by the observer.  For all comments but the
                // syntactic, use the comment text to create new pseudo
                // utterances by combining tokens from the utterance with
                // pseudo-tokens in the comment.  The pseudo-tokens have a
                // "-GROUNDING" suffix which distiguishes them from tokens
                // actually present in the uttered speech.
                NodeList auxNodes = item.getElementsByTagName("a");
                List<String> augmentedUtterances = new LinkedList<String>();
                if (generateAugmented) {
                    for (int j = 0; j < auxNodes.getLength(); ++j) {
                        // Get any comment for the utterance
                        Node n = auxNodes.item(j);
                        String auxNodeType = n.getAttributes().
                            getNamedItem("type").getNodeValue();

                        // Get only those nodes that contain comments or
                        // descriptions on the utterance that may be used to
                        // ground the words being referred to.
                        if (auxNodeType.equals("action")
                            || auxNodeType.equals("actions")
                            || auxNodeType.equals("addressee")
                            || auxNodeType.equals("comments")
                            || auxNodeType.equals("explanation")
                            || auxNodeType.equals("gesture")
                            || auxNodeType.equals("happening")
                            || auxNodeType.equals("situation")) {
                            
                            String commentOnUtterance = 
                                n.getFirstChild().getNodeValue();
                            // Use the iterator factory to tokenize in the event
                            // that the user has specified some form of token
                            // filtering
                            Iterator<String> tokenIter = 
                                IteratorFactory.tokenize(
                                        new BufferedReader(
                                            new StringReader(
                                                commentOnUtterance)));
                            // For each of the tokens in the additional
                            // information, create a pseudo-utterance using a
                            // word from the actual utterance and an
                            // grounding-token
                            while (tokenIter.hasNext()) {
                                String token = tokenIter.next();
                                for (String word : wordsInUtterance) {
                                    augmentedUtterances.add(word + " "
                                        + token + "-GROUNDING");
                                }
                            }
                        }
                    }
                }
                    
                // Write the utterance if an utterance is a document.
                for (Iterator<String> it = wordsInUtterance.iterator(); 
                         it.hasNext(); ) {
                    utteranceBuilder.append(it.next());
                    if (it.hasNext())
                        utteranceBuilder.append(" ");
                }
                String utterance = utteranceBuilder.toString();
                if (utterancePerDoc) {
                    print(utterance);
                    // Print all the psuedo utterances constructed from the
                    // comments
                    for (String aug : augmentedUtterances)
                        print(aug);
                }
                else {  // otherwise save the utterance.
                    fileBuilder.append(utterance);
                    if (separateByPeriod)
                        fileBuilder.append(".");
                    fileBuilder.append(" ");

                    // Print all the psuedo utterances constructed from the
                    // comments.  Unlike the utterances, print these as separate
                    // documents to avoid having them register as co-occurrences
                    // with other utterances.
                    for (String aug : augmentedUtterances)
                        print(aug);
                }
            }

            // Write all the utterances if the whole xml file is a document.
            if (!utterancePerDoc)
                print(fileBuilder.toString());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }