Create ngram dictionary from training data.

@param aDictSamples
the training data
@param aNgramCutoff
the cutoff
@return ngram dictionary
428458
protected final Dictionary createNgramDictionary(
      final ObjectStream<POSSample> aDictSamples, final int aNgramCutoff) {
    Dictionary ngramDict = null;
    if (aNgramCutoff != Flags.DEFAULT_DICT_CUTOFF) {
      System.err.print("Building ngram dictionary ... ");
      try {
        ngramDict = POSTaggerME
            .buildNGramDictionary(aDictSamples, aNgramCutoff);
        this.dictSamples.reset();
      } catch (final IOException e) {
        throw new TerminateToolException(-1,
            "IO error while building NGram Dictionary: " + e.getMessage(), e);
      }
      System.err.println("done");
    }
    return ngramDict;
  }
163510
def extract_ngram_filter(pos_seq, regex=SimpleNP, minlen=1, maxlen=8):
	""""""
	ss = coarse_tag_str(pos_seq)

	def gen():
		for s in xrange(len(ss)):
			for n in xrange(minlen, 1 + min(maxlen, len(ss) - s)):
				e = s + n
				substr = ss[s:e]
				if re.match(regex + "$", substr):
					yield (s, e)

	return list(gen())
331461
def generate_encoded_samples(self, data_dir, tmp_dir, dataset_split):
    
    generator = self.generate_samples(data_dir, tmp_dir, dataset_split)
    encoder = self.get_or_create_vocab(data_dir, tmp_dir)
    label_encoder = self.get_labels_encoder(data_dir)
    for sample in generator:
      inputs = encoder.encode(sample["inputs"])
      inputs.append(text_encoder.EOS_ID)
      context = encoder.encode(sample["context"])
      context.append(text_encoder.EOS_ID)
      targets = label_encoder.encode(sample["targets"])
      sample["targets"] = targets
      yield {"inputs": inputs, "context": context, "targets": targets}
1185379
function cloneListForMapAndSample(original, excludeDimensions) {
    var allDimensions = original.dimensions;
    var list = new List(
        zrUtil.map(allDimensions, original.getDimensionInfo, original),
        original.hostModel
    );
    // FIXME If needs stackedOn, value may already been stacked
    transferProperties(list, original);

    var storage = list._storage = {};
    var originalStorage = original._storage;

    // Init storage
    for (var i = 0; i < allDimensions.length; i++) {
        var dim = allDimensions[i];
        if (originalStorage[dim]) {
            // Notice that we do not reset invertedIndicesMap here, becuase
            // there is no scenario of mapping or sampling ordinal dimension.
            if (zrUtil.indexOf(excludeDimensions, dim) >= 0) {
                storage[dim] = cloneDimStore(originalStorage[dim]);
                list._rawExtent[dim] = getInitialExtent();
                list._extent[dim] = null;
            }
            else {
                // Direct reference for other dimensions
                storage[dim] = originalStorage[dim];
            }
        }
    }
    return list;
}
1373788
static DirectCompactDoublesSketch createFromUpdateSketch(final UpdateDoublesSketch sketch,
                                                           final WritableMemory dstMem) {
    final long memCap = dstMem.getCapacity();
    final int k = sketch.getK();
    final long n = sketch.getN();
    checkDirectMemCapacity(k, n, memCap);

    //initialize dstMem
    dstMem.putLong(0, 0L); //clear pre0
    insertPreLongs(dstMem, 2);
    insertSerVer(dstMem, DoublesSketch.DOUBLES_SER_VER);
    insertFamilyID(dstMem, Family.QUANTILES.getID());
    insertK(dstMem, k);

    final int flags = COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK; // true for all compact sketches

    if (sketch.isEmpty()) {
      insertFlags(dstMem, flags | EMPTY_FLAG_MASK);
    } else {
      insertFlags(dstMem, flags);
      insertN(dstMem, n);
      insertMinDouble(dstMem, sketch.getMinValue());
      insertMaxDouble(dstMem, sketch.getMaxValue());

      final int bbCount = computeBaseBufferItems(k, n);

      final DoublesSketchAccessor inputAccessor = DoublesSketchAccessor.wrap(sketch);
      assert bbCount == inputAccessor.numItems();

      long dstMemOffset = COMBINED_BUFFER;

      // copy and sort base buffer
      final double[] bbArray = inputAccessor.getArray(0, bbCount);
      Arrays.sort(bbArray);
      dstMem.putDoubleArray(dstMemOffset, bbArray, 0, bbCount);
      dstMemOffset += bbCount << 3;

      long bitPattern = computeBitPattern(k, n);
      for (int lvl = 0; bitPattern > 0; ++lvl, bitPattern >>>= 1) {
        if ((bitPattern & 1L) > 0L) {
          inputAccessor.setLevel(lvl);
          dstMem.putDoubleArray(dstMemOffset, inputAccessor.getArray(0, k), 0, k);
          dstMemOffset += k << 3;
        }
      }
    }

    final DirectCompactDoublesSketch dcds = new DirectCompactDoublesSketch(k);
    dcds.mem_ = dstMem;

    return dcds;
  }
49927
private double[] detectBlockLongText(List<String> ngrams) {
        assert !ngrams.isEmpty();
        double[] langprob = new double[ngramFrequencyData.getLanguageList().size()];
        Random rand = new Random(seed.or(DEFAULT_SEED));
        for (int t = 0; t < N_TRIAL; ++t) {
            double[] prob = initProbability();
            double alpha = this.alpha + (rand.nextGaussian() * ALPHA_WIDTH);

            for (int i=0; i<ITERATION_LIMIT; i++) {
                int r = rand.nextInt(ngrams.size());
                updateLangProb(prob, ngrams.get(r), 1, alpha);
                if (i % 5 == 0) {
                    if (Util.normalizeProb(prob) > CONV_THRESHOLD) break; //this break ensures that we quit the loop before all probabilities reach 0
                    if (logger.isTraceEnabled()) logger.trace("> " + sortProbability(prob));
                }
            }
            for(int j=0;j<langprob.length;++j) langprob[j] += prob[j] / N_TRIAL;
            if (logger.isDebugEnabled()) logger.debug("==> " + sortProbability(prob));
        }
        return langprob;
    }
1289848
static DirectCompactUnorderedSketch compact(final UpdateSketch sketch,
      final WritableMemory dstMem) {
    final int curCount = sketch.getRetainedEntries(true);
    long thetaLong = sketch.getThetaLong();
    boolean empty = sketch.isEmpty();
    thetaLong = thetaOnCompact(empty, curCount, thetaLong);
    empty = emptyOnCompact(curCount, thetaLong);
    final int preLongs = computeCompactPreLongs(thetaLong, empty, curCount);
    final short seedHash = sketch.getSeedHash();
    final long[] cache = sketch.getCache();
    final int requiredFlags = READ_ONLY_FLAG_MASK | COMPACT_FLAG_MASK;
    final byte flags = (byte) (requiredFlags | (empty ? EMPTY_FLAG_MASK : 0));
    final boolean ordered = false;
    final long[] compactCache = CompactSketch.compactCache(cache, curCount, thetaLong, ordered);
    loadCompactMemory(compactCache, seedHash, curCount, thetaLong, dstMem, flags, preLongs);
    return new DirectCompactUnorderedSketch(dstMem);
  }
1032073
def create_affine(sorted_dicoms):
    

    # Create affine matrix (http://nipy.sourceforge.net/nibabel/dicom/dicom_orientation.html#dicom-slice-affine)
    image_orient1 = numpy.array(sorted_dicoms[0].ImageOrientationPatient)[0:3]
    image_orient2 = numpy.array(sorted_dicoms[0].ImageOrientationPatient)[3:6]

    delta_r = float(sorted_dicoms[0].PixelSpacing[0])
    delta_c = float(sorted_dicoms[0].PixelSpacing[1])

    image_pos = numpy.array(sorted_dicoms[0].ImagePositionPatient)

    last_image_pos = numpy.array(sorted_dicoms[-1].ImagePositionPatient)

    if len(sorted_dicoms) == 1:
        # Single slice
        step = [0, 0, -1]
    else:
        step = (image_pos - last_image_pos) / (1 - len(sorted_dicoms))

    # check if this is actually a volume and not all slices on the same location
    if numpy.linalg.norm(step) == 0.0:
        raise ConversionError("NOT_A_VOLUME")

    affine = numpy.array(
        [[-image_orient1[0] * delta_c, -image_orient2[0] * delta_r, -step[0], -image_pos[0]],
         [-image_orient1[1] * delta_c, -image_orient2[1] * delta_r, -step[1], -image_pos[1]],
         [image_orient1[2] * delta_c, image_orient2[2] * delta_r, step[2], image_pos[2]],
         [0, 0, 0, 1]]
    )
    return affine, numpy.linalg.norm(step)
1348041
protected Sample createSampleObject(List<? extends ByteBuffer> nals) {
        byte[] sizeInfo = new byte[nals.size() * 4];
        ByteBuffer sizeBuf = ByteBuffer.wrap(sizeInfo);
        for (ByteBuffer b : nals) {
            sizeBuf.putInt(b.remaining());
        }

        ByteBuffer[] data = new ByteBuffer[nals.size() * 2];

        for (int i = 0; i < nals.size(); i++) {
            data[2 * i] = ByteBuffer.wrap(sizeInfo, i * 4, 4);
            data[2 * i + 1] = nals.get(i);
        }

        return new SampleImpl(data, getCurrentSampleEntry());
    }
144186
def _render_ngram_row(self, ngram, ngram_group, row_template, labels):
        """"""
        cell_data = {'ngram': ngram}
        label_data = {}
        for label in labels:
            label_data[label] = []
        work_grouped = ngram_group.groupby(constants.WORK_FIELDNAME)
        for work, group in work_grouped:
            min_count = group[constants.COUNT_FIELDNAME].min()
            max_count = group[constants.COUNT_FIELDNAME].max()
            if min_count == max_count:
                count = min_count
            else:
                count = '{}\N{EN DASH}{}'.format(min_count, max_count)
            label_data[group[constants.LABEL_FIELDNAME].iloc[0]].append(
                '{} ({})'.format(work, count))
        for label, data in label_data.items():
            label_data[label] = '; '.join(data)
        cell_data.update(label_data)
        html = row_template.format(**cell_data)
        return '<tr>\n{}\n</tr>'.format(html)
884530
def ngram2mgram(input_file, output_file, n, m, words=False, ascii_idngram=False):
    
    cmd = ['ngram2mgram', '-n', n,
                          '-m', m]

    if words and ascii_idngram:
        raise ConversionError("Parameters 'words' and 'ascii_idngram' cannot both be True")

    if words:
        cmd.append('-words')
    elif ascii_idngram:
        cmd.append('-ascii')
    else:
        cmd.append('-binary')

    # Ensure that every parameter is of type 'str'
    cmd = [str(x) for x in cmd]

    with open(input_file,'r') as input_f:
        with open(output_file,'w+') as output_f:
            with  output_to_debuglogger() as err_f:
                exitcode = subprocess.call(cmd, stdin=input_f, stdout=output_f, stderr=err_f)
    
    logger = logging.getLogger(__name__)
    logger.debug("Command '%s' returned with exit code '%d'." % (' '.join(cmd), exitcode))

    if exitcode != 0:
        raise ConversionError("'%s' returned with non-zero exit status '%s'" % (cmd[0], exitcode))
839639
public ArrayOfDoublesUpdatableSketch build(final WritableMemory dstMem) {
    return new DirectArrayOfDoublesQuickSelectSketch(nomEntries_, resizeFactor_.lg(), 
        samplingProbability_, numValues_, seed_, dstMem);
  }
1372778
private static void ngramOnCharacter(String data, int[] gramSizes, Collection<String> list) {
    	data = data.replaceAll("\\s+", "");
    	for (int j = 0; j < gramSizes.length; j++) {
            int len = gramSizes[j];
            if (len <= 0 || len > data.length())
                continue;
            for (int i = 0; i < data.length() - len; i++) {
                list.add(data.substring(i, i + len));
            }
        }
    }
49865
def generate_slices(slices, minsize, maxsize, bounded=True):
  """"""

  if isinstance(slices, integer_types) or isinstance(slices, floating_types):
    slices = [ slice(int(slices), int(slices)+1, 1) ]
  elif type(slices) == slice:
    slices = [ slices ]
  elif slices == Ellipsis:
    slices = []

  slices = list(slices)

  for index, slc in enumerate(slices):
    if slc == Ellipsis:
      fill = len(maxsize) - len(slices) + 1
      slices = slices[:index] +  (fill * [ slice(None, None, None) ]) + slices[index+1:]
      break

  while len(slices) < len(maxsize):
    slices.append( slice(None, None, None) )

  # First three slices are x,y,z, last is channel. 
  # Handle only x,y,z here, channel seperately
  for index, slc in enumerate(slices):
    if isinstance(slc, integer_types) or isinstance(slc, floating_types):
      slices[index] = slice(int(slc), int(slc)+1, 1)
    elif slc == Ellipsis:
      raise ValueError("More than one Ellipsis operator used at once.")
    else:
      start = minsize[index] if slc.start is None else slc.start
      end = maxsize[index] if slc.stop is None else slc.stop 
      step = 1 if slc.step is None else slc.step

      if step < 0:
        raise ValueError('Negative step sizes are not supported. Got: {}'.format(step))

      # note: when unbounded, negative indicies do not refer to
      # the end of the volume as they can describe, e.g. a 1px
      # border on the edge of the beginning of the dataset as in
      # marching cubes.
      if bounded:
        # if start < 0: # this is support for negative indicies
          # start = maxsize[index] + start         
        check_bounds(start, minsize[index], maxsize[index])
        # if end < 0: # this is support for negative indicies
        #   end = maxsize[index] + end
        check_bounds(end, minsize[index], maxsize[index])

      slices[index] = slice(start, end, step)

  return slices
88454
def _get_ngram_counter(ids, n):
  
  # Remove zero IDs used to pad the sequence.
  ids = [token_id for token_id in ids if token_id != 0]
  ngram_list = [tuple(ids[i:i + n]) for i in range(len(ids) + 1 - n)]
  ngrams = set(ngram_list)
  counts = collections.Counter()
  for ngram in ngrams:
    counts[ngram] = 1
  return counts
1019285
static public function readCombFreq(string $sep, array $ngrams) : array
    {
        $ngrams = self::ngramsAsArray($sep, $ngrams);
        $frequencies = array();
        foreach ($ngrams as $ngram) {
            foreach ($ngram as $posToken => $token) {
                for ($i = $posToken+1; $i < count($ngram); $i++) {
                    if(isset($frequencies[$token.$sep.$ngram[$i]][$posToken][$i])) { //checks if the combo already exists
                        $frequencies[$token.$sep.$ngram[$i]][$posToken][$i] += 1;
                    } else {
                        $frequencies[$token.$sep.$ngram[$i]][$posToken][$i] = 1;
                    }
                }
            }

        }
        return $frequencies;
    }
986808
public static Map<String, Integer> createNGrams(String inputQuery, boolean removeStopWords) {
    List<String> wordsInString =
        Lists.newArrayList(Stemmer.replaceIllegalCharacter(inputQuery).split(" "));
    if (removeStopWords) wordsInString.removeAll(STOPWORDSLIST);
    List<String> stemmedWordsInString =
        wordsInString.stream().map(Stemmer::stem).collect(Collectors.toList());
    Map<String, Integer> tokens = new HashMap<>();
    // Padding the string
    for (String singleWord : stemmedWordsInString) {
      if (!StringUtils.isEmpty(singleWord)) {
        // The s$ will be the produced from two words.
        StringBuilder singleString = new StringBuilder(singleWord.length() + 2);
        singleString.append('^').append(singleWord.toLowerCase()).append('$');
        int length = singleString.length();
        for (int i = 0; i < length - 1; i++) {
          String token = null;
          if (i + N_GRAMS < length) {
            token = singleString.substring(i, i + N_GRAMS);

          } else {
            token = singleString.substring(length - 2);
          }

          if (!tokens.containsKey(token)) {
            tokens.put(token, 1);
          } else {
            tokens.put(token, (tokens.get(token) + 1));
          }
        }
      }
    }

    return tokens;
  }