http://stackoverflow.com/questions/23903113/mapreduce-error-usergroupinformation-priviledgedactionexception
971406
protected static String getInstancesFolder(FileSystem fS, Configuration conf) throws IOException {
    return conf.get(HDFS_TMP_FOLDER_CONF, fS.equals(FileSystem.getLocal(conf)) ? DEFAULT_LOCAL_TMP_FOLDER_CONF_VALUE
        : DEFAULT_HDFS_TMP_FOLDER_CONF_VALUE);
  }
317208
static List<PartitionDefinition> findSlicePartitions(RestClient client, Settings settings, MappingSet mappingSet,
                                                         Map<String, NodeInfo> nodes, List<List<Map<String, Object>>> shards, Log log) {
        QueryBuilder query = QueryUtils.parseQueryAndFilters(settings);
        Integer maxDocsPerPartition = settings.getMaxDocsPerPartition();
        Assert.notNull(maxDocsPerPartition, "Attempting to find slice partitions but maximum documents per partition is not set.");
        Resource readResource = new Resource(settings, true);
        Mapping resolvedMapping = mappingSet == null ? null : mappingSet.getResolvedView();

        List<PartitionDefinition> partitions = new ArrayList<PartitionDefinition>(shards.size());
        for (List<Map<String, Object>> group : shards) {
            String index = null;
            int shardId = -1;
            List<String> locationList = new ArrayList<String> ();
            for (Map<String, Object> replica : group) {
                ShardInfo shard = new ShardInfo(replica);
                index = shard.getIndex();
                shardId = shard.getName();
                if (nodes.containsKey(shard.getNode())) {
                    locationList.add(nodes.get(shard.getNode()).getPublishAddress());
                }
            }
            String[] locations = locationList.toArray(new String[0]);
            if (index == null) {
                // Could not find shards for this partition. Continue anyway?
                if (settings.getIndexReadAllowRedStatus()) {
                    log.warn("Shard information is missing from an index and will not be reached during job execution. " +
                            "Assuming shard is unavailable and cluster is red! Continuing with read operation by " +
                            "skipping this shard! This may result in incomplete data retrieval!");
                } else {
                    throw new IllegalStateException("Could not locate shard information for one of the read indices. " +
                            "Check your cluster status to see if it is unstable!");
                }
            } else {
                // TODO applyAliasMetaData should be called in order to ensure that the count are exact (alias filters and routing may change the number of documents)
                long numDocs;
                if (readResource.isTyped()) {
                    numDocs = client.count(index, readResource.type(), Integer.toString(shardId), query);
                } else {
                    numDocs = client.countIndexShard(index, Integer.toString(shardId), query);
                }
                int numPartitions = (int) Math.max(1, numDocs / maxDocsPerPartition);
                for (int i = 0; i < numPartitions; i++) {
                    PartitionDefinition.Slice slice = new PartitionDefinition.Slice(i, numPartitions);
                    partitions.add(new PartitionDefinition(settings, resolvedMapping, index, shardId, slice, locations));
                }
            }
        }
        return partitions;
    }
3319
@Override
    @Path("/{groupName}/{'users'}/{cuid}")
    @ApiOperation(value="Create a workgroup or add a user to an existing workgroup",
        notes="If users/{cuid} is present, user is added to workgroup.", response=StatusMessage.class)
    @ApiImplicitParams({
        @ApiImplicitParam(name="Workgroup", paramType="body", required=true, dataType="com.centurylink.mdw.model.user.Workgroup"),
        @ApiImplicitParam(name="groupName", paramType="path", required=true)})
    public JSONObject post(String path, JSONObject content, Map<String,String> headers)
    throws ServiceException, JSONException {
        String name = getSegment(path, 1);
        String rel = getSegment(path, 2);

        UserServices userServices = ServiceLocator.getUserServices();
        try {
            if (rel == null) {
                Workgroup workgroup = new Workgroup(content);
                Workgroup existing = userServices.getWorkgroups().get(workgroup.getName());
                if (existing != null)
                    throw new ServiceException(HTTP_409_CONFLICT, "Workgroup name already exists: " + workgroup.getName());
                userServices.createWorkgroup(workgroup);
                headers.put(Listener.METAINFO_HTTP_STATUS_CODE, "201");
            }
            else if (rel.equals("users")) {
                String cuid = getSegment(path, 3);
                User user = UserGroupCache.getUser(cuid);
                if (user == null) {
                    throw new CachingException("Cannot find user: " + cuid);
                }
                if (user.belongsToGroup(name))  // in case added elsewhere
                    throw new ServiceException(HTTP_409_CONFLICT, "User " + cuid + " already belongs to workgroup " + name);

                userServices.addUserToWorkgroup(cuid, name);
            }
            else {
                String msg = "Unsupported relationship for workgroup " + name + ": " + rel;
                throw new ServiceException(HTTP_400_BAD_REQUEST, msg);
            }
            return null;
        }
        catch (DataAccessException ex) {
            throw new ServiceException(HTTP_500_INTERNAL_ERROR, ex.getMessage(), ex);
        }
        catch (CachingException ex) {
             throw new ServiceException(HTTP_500_INTERNAL_ERROR, ex.getMessage(), ex);
        }
    }
734853
private void parseAndAddMapTaskCounters(MapTaskStatistics mapTask, String counters) throws ParseException {
    Counters cnt = Counters.fromEscapedCompactString(counters);
    for (java.util.Iterator<Counters.Group> grps = cnt.iterator(); grps.hasNext(); ) {
      Counters.Group grp = grps.next();
      //String groupname = "<" + grp.getName() + ">::<" + grp.getDisplayName() + ">";
      for (java.util.Iterator<Counters.Counter> mycounters = grp.iterator(); mycounters.hasNext(); ) {
        Counters.Counter counter = mycounters.next();
        //String countername = "<"+counter.getName()+">::<"+counter.getDisplayName()+">::<"+counter.getValue()+">";
        //System.out.println("groupName:"+groupname+",countername: "+countername);
        String countername = grp.getDisplayName()+"."+counter.getDisplayName();
        String value = (new Long(counter.getValue())).toString();
        String[] parts = {countername,value};
        //System.out.println("part0:"+parts[0]+",:part1 "+parts[1]);
        if (parts[0].equals("FileSystemCounters.FILE_BYTES_READ")) {
          mapTask.setValue(MapTaskKeys.FILE_BYTES_READ, parts[1]);
        } else if (parts[0].equals("FileSystemCounters.FILE_BYTES_WRITTEN")) {
          mapTask.setValue(MapTaskKeys.FILE_BYTES_WRITTEN, parts[1]);
        } else if (parts[0].equals("FileSystemCounters.HDFS_BYTES_READ")) {
          mapTask.setValue(MapTaskKeys.HDFS_BYTES_READ, parts[1]);
        } else if (parts[0].equals("FileSystemCounters.HDFS_BYTES_WRITTEN")) {
          mapTask.setValue(MapTaskKeys.HDFS_BYTES_WRITTEN, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Map input records")) {
          mapTask.setValue(MapTaskKeys.INPUT_RECORDS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Map output records")) {
          mapTask.setValue(MapTaskKeys.OUTPUT_RECORDS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Map input bytes")) {
          mapTask.setValue(MapTaskKeys.INPUT_BYTES, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Map output bytes")) {
          mapTask.setValue(MapTaskKeys.OUTPUT_BYTES, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Combine input records")) {
          mapTask.setValue(MapTaskKeys.COMBINE_INPUT_RECORDS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Combine output records")) {
          mapTask.setValue(MapTaskKeys.COMBINE_OUTPUT_RECORDS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Spilled Records")) {
          mapTask.setValue(MapTaskKeys.SPILLED_RECORDS, parts[1]);
        } else {
          System.err.println("MapCounterKey:<"+parts[0]+"> ==> NOT INCLUDED IN PERFORMANCE ADVISOR MAP TASK");
        }
      }    
    }
  }
852429
private void populateMap(final int[] priKeyIndices) {
    final SketchIterator<ArrayOfStringsSummary> it = sketch.iterator();
    Arrays.fill(hashArr, 0L);
    Arrays.fill(priKeyArr, null);
    Arrays.fill(counterArr, 0);
    groupCount = 0;
    final int lgMapArrSize = Integer.numberOfTrailingZeros(mapArrSize);

    while (it.next()) {
      final String[] arr = it.getSummary().getValue();
      final String priKey = getPrimaryKey(arr, priKeyIndices, sep);
      final long hash = stringHash(priKey);
      final int index = hashSearchOrInsert(hashArr, lgMapArrSize, hash);
      if (index < 0) { //was empty, hash inserted
        final int idx = -(index + 1); //actual index
        counterArr[idx] = 1;
        groupCount++;
        priKeyArr[idx] = priKey;
      } else { //found, duplicate
        counterArr[index]++; //increment
      }
    }
    mapValid = true;
  }
101113
public static void createFileBatchesSpark(JavaRDD<String> filePaths, final String rootOutputDir, final int batchSize, @NonNull final org.apache.hadoop.conf.Configuration hadoopConfig) {
        final SerializableHadoopConfig conf = new SerializableHadoopConfig(hadoopConfig);
        //Here: assume input is images. We can't store them as Float32 arrays - that's too inefficient
        // instead: let's store the raw file content in a batch.
        long count = filePaths.count();
        long maxPartitions = count / batchSize;
        JavaRDD<String> repartitioned = filePaths.repartition(Math.max(filePaths.getNumPartitions(), (int) maxPartitions));
        repartitioned.foreachPartition(new VoidFunction<Iterator<String>>() {
            @Override
            public void call(Iterator<String> stringIterator) throws Exception {
                //Construct file batch
                List<String> list = new ArrayList<>();
                List<byte[]> bytes = new ArrayList<>();
                FileSystem fs = FileSystem.get(conf.getConfiguration());
                while (stringIterator.hasNext()) {
                    String inFile = stringIterator.next();
                    byte[] fileBytes;
                    try (BufferedInputStream bis = new BufferedInputStream(fs.open(new Path(inFile)))) {
                        fileBytes = IOUtils.toByteArray(bis);
                    }
                    list.add(inFile);
                    bytes.add(fileBytes);

                    if (list.size() == batchSize) {
                        process(list, bytes);
                    }
                }
                if (list.size() > 0) {
                    process(list, bytes);
                }
            }

            private void process(List<String> paths, List<byte[]> bytes) throws IOException {
                FileBatch fb = new FileBatch(bytes, paths);
                String name = UUID.randomUUID().toString().replaceAll("-", "") + ".zip";
                String outPath = FilenameUtils.concat(rootOutputDir, name);
                FileSystem fileSystem = FileSystem.get(conf.getConfiguration());
                try (BufferedOutputStream bos = new BufferedOutputStream(fileSystem.create(new Path(outPath)))) {
                    fb.writeAsZip(bos);
                }

                paths.clear();
                bytes.clear();
            }
        });
    }
1260876
private double toPerPartitionMaxSize(int maxConfiguredSize, String mapName) {
        int memberCount = clusterService.getSize(DATA_MEMBER_SELECTOR);
        double translatedPartitionSize = (1D * maxConfiguredSize * memberCount / partitionCount);

        if (translatedPartitionSize < 1) {
            translatedPartitionSize = MIN_TRANSLATED_PARTITION_SIZE;
            logMisconfiguredPerNodeMaxSize(mapName, memberCount);
        }

        return translatedPartitionSize;
    }
439397
@Override
    GroupReplyList write_attribute_reply_i(final int rid, final int tmo, final boolean fwd) throws DevFailed {
	final Integer rid_obj = new Integer(rid);
	final GroupReplyList rl = new GroupReplyList();
	final AsynchRequest ar = arp.get(rid_obj);
	if (ar == null) {
	    final DevError[] errors = new DevError[1];
	    errors[0] = new DevError();
	    errors[0].severity = ErrSeverity.ERR;
	    errors[0].reason = "API_BadAsynPollId";
	    errors[0].desc = "Invalid asynch. request identifier specified";
	    errors[0].origin = "GroupDeviceElement.write_attribute_reply";
	    final DevFailed e = new DevFailed(errors);
	    rl.add(new GroupReply(get_name(), "unknown", e));
	    return rl;
	}
	if (ar.req_id == -1) {
	    for (final String element : ar.obj_name) {
		rl.add(new GroupReply(get_name(), element, ar.exception));
	    }
	    arp.remove(rid_obj);
	    return rl;
	}
	try {
	    proxy.write_attribute_reply(ar.req_id, tmo);
	    for (final String element : ar.obj_name) {
		rl.add(new GroupReply(get_name(), element));
	    }
	} catch (final AsynReplyNotArrived na) {
	    final DevError[] errors = new DevError[1];
	    errors[0] = new DevError();
	    errors[0].severity = ErrSeverity.ERR;
	    errors[0].reason = "API_AsynReplyNotArrived";
	    errors[0].desc = "No reply for asynch request";
	    errors[0].origin = "GroupDeviceElement.write_attribute_reply";
	    final DevFailed e = new DevFailed(errors);
	    for (final String element : ar.obj_name) {
		rl.add(new GroupReply(get_name(), element, e));
	    }
	} catch (final DevFailed df) {
	    for (final String element : ar.obj_name) {
		rl.add(new GroupReply(get_name(), element, df));
	    }
	} catch (final Exception ex) {
	    final DevError[] errors = new DevError[1];
	    errors[0] = new DevError();
	    errors[0].severity = ErrSeverity.ERR;
	    errors[0].reason = "unknown exception caught";
	    errors[0].desc = "unknown error";
	    errors[0].origin = "GroupDeviceElemnt.write_attribute_reply";
	    final DevFailed e = new DevFailed(errors);
	    for (final String element : ar.obj_name) {
		rl.add(new GroupReply(get_name(), element, e));
	    }
	}
	arp.remove(rid_obj);
        return rl;
    }
880224
@Override
    public ILockableEntityGroup findGroupWithLock(String key, String owner) throws GroupsException {
        return findGroupWithLock(key, owner, 0);
    }
1015508
function setHashOrThrowError(parseMap, routeObject) {
    var route = routeObject.route;
    var get = routeObject.get;
    var set = routeObject.set;
    var call = routeObject.call;

    getHashesFromRoute(route).
        map(function mapHashToString(hash) { return hash.join(','); }).
        forEach(function forEachRouteHash(hash) {
            if (get && parseMap[hash + 'get'] ||
                set && parseMap[hash + 'set'] ||
                    call && parseMap[hash + 'call']) {
                throw new Error(errors.routeWithSamePrecedence + ' ' +
                               prettifyRoute(route));
            }
            if (get) {
                parseMap[hash + 'get'] = true;
            }
            if (set) {
                parseMap[hash + 'set'] = true;
            }
            if (call) {
                parseMap[hash + 'call'] = true;
            }
        });
}
267267
@Override
	public void prepare() throws Exception {
		final TaskConfig config = this.taskContext.getTaskConfig();
		this.strategy = config.getDriverStrategy();
		
		if (strategy == DriverStrategy.ALL_GROUP_COMBINE) {
			if (!(this.taskContext.getStub() instanceof GenericCombine)) {
				throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + GenericCombine.class.getName());
			}
		}
		else if (strategy != DriverStrategy.ALL_GROUP_REDUCE) {
			throw new Exception("Unrecognized driver strategy for AllGroupReduce driver: " + config.getDriverStrategy().name());
		}
		this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
		this.input = this.taskContext.getInput(0);
	}
931676
@Override
	public void mTokens() throws RecognitionException {
		// druidG.g:1:8: ( T__89 | T__90 | T__91 | T__92 | LPARAN | RPARAN | LCURLY | RCURLY | LSQUARE | RSQUARE | INSERT | INSERT_HADOOP | INSERT_REALTIME | INTO | VALUES | MAX_WINDOW | DELIMITER | PARTITION | ROLLUP | DROP | TABLE | DELETE | KAFKA | STRING | ISO | AUTO_ISO | SELECT | COUNT | LONG_SUM | DOUBLE_SUM | UNIQUE | HYPER_UNIQUE | MIN | MAX | DURATION | PERIOD | INCLUDE | WHICH | CONTAINS | SORT | HINT | AS | FIELD_ACCESS | FROM | WHERE | BETWEEN | AND | OR | NOT | GROUP | ASC | DESC | ORDER | HAVING | BREAK | BY | LIMIT | LIKE | THEN | JAVASCRIPT | JOIN | LEFT_JOIN | RIGHT_JOIN | ON | OPT_SEMI_COLON | OPT_AMPERSAND | WS | DATE_YEAR_MONTH_ONLY | DATE | DATE_HOUR | DATE_HOUR_MIN | DATE_HOUR_MIN_SEC | DATE_HOUR_MIN_SEC_SUB | DATE_HOUR_MIN_SEC_SUB_TZ | DATE_HOUR_MIN_SEC_SUB_UTC_TZ | ARITH_OPER | EQUALS | COMPARE_OPER | ID | SINGLE_QUOTE_STRING | LONG | FLOAT )
		int alt47=82;
		alt47 = dfa47.predict(input);
		switch (alt47) {
			case 1 :
				// druidG.g:1:10: T__89
				{
				mT__89(); 

				}
				break;
			case 2 :
				// druidG.g:1:16: T__90
				{
				mT__90(); 

				}
				break;
			case 3 :
				// druidG.g:1:22: T__91
				{
				mT__91(); 

				}
				break;
			case 4 :
				// druidG.g:1:28: T__92
				{
				mT__92(); 

				}
				break;
			case 5 :
				// druidG.g:1:34: LPARAN
				{
				mLPARAN(); 

				}
				break;
			case 6 :
				// druidG.g:1:41: RPARAN
				{
				mRPARAN(); 

				}
				break;
			case 7 :
				// druidG.g:1:48: LCURLY
				{
				mLCURLY(); 

				}
				break;
			case 8 :
				// druidG.g:1:55: RCURLY
				{
				mRCURLY(); 

				}
				break;
			case 9 :
				// druidG.g:1:62: LSQUARE
				{
				mLSQUARE(); 

				}
				break;
			case 10 :
				// druidG.g:1:70: RSQUARE
				{
				mRSQUARE(); 

				}
				break;
			case 11 :
				// druidG.g:1:78: INSERT
				{
				mINSERT(); 

				}
				break;
			case 12 :
				// druidG.g:1:85: INSERT_HADOOP
				{
				mINSERT_HADOOP(); 

				}
				break;
			case 13 :
				// druidG.g:1:99: INSERT_REALTIME
				{
				mINSERT_REALTIME(); 

				}
				break;
			case 14 :
				// druidG.g:1:115: INTO
				{
				mINTO(); 

				}
				break;
			case 15 :
				// druidG.g:1:120: VALUES
				{
				mVALUES(); 

				}
				break;
			case 16 :
				// druidG.g:1:127: MAX_WINDOW
				{
				mMAX_WINDOW(); 

				}
				break;
			case 17 :
				// druidG.g:1:138: DELIMITER
				{
				mDELIMITER(); 

				}
				break;
			case 18 :
				// druidG.g:1:148: PARTITION
				{
				mPARTITION(); 

				}
				break;
			case 19 :
				// druidG.g:1:158: ROLLUP
				{
				mROLLUP(); 

				}
				break;
			case 20 :
				// druidG.g:1:165: DROP
				{
				mDROP(); 

				}
				break;
			case 21 :
				// druidG.g:1:170: TABLE
				{
				mTABLE(); 

				}
				break;
			case 22 :
				// druidG.g:1:176: DELETE
				{
				mDELETE(); 

				}
				break;
			case 23 :
				// druidG.g:1:183: KAFKA
				{
				mKAFKA(); 

				}
				break;
			case 24 :
				// druidG.g:1:189: STRING
				{
				mSTRING(); 

				}
				break;
			case 25 :
				// druidG.g:1:196: ISO
				{
				mISO(); 

				}
				break;
			case 26 :
				// druidG.g:1:200: AUTO_ISO
				{
				mAUTO_ISO(); 

				}
				break;
			case 27 :
				// druidG.g:1:209: SELECT
				{
				mSELECT(); 

				}
				break;
			case 28 :
				// druidG.g:1:216: COUNT
				{
				mCOUNT(); 

				}
				break;
			case 29 :
				// druidG.g:1:222: LONG_SUM
				{
				mLONG_SUM(); 

				}
				break;
			case 30 :
				// druidG.g:1:231: DOUBLE_SUM
				{
				mDOUBLE_SUM(); 

				}
				break;
			case 31 :
				// druidG.g:1:242: UNIQUE
				{
				mUNIQUE(); 

				}
				break;
			case 32 :
				// druidG.g:1:249: HYPER_UNIQUE
				{
				mHYPER_UNIQUE(); 

				}
				break;
			case 33 :
				// druidG.g:1:262: MIN
				{
				mMIN(); 

				}
				break;
			case 34 :
				// druidG.g:1:266: MAX
				{
				mMAX(); 

				}
				break;
			case 35 :
				// druidG.g:1:270: DURATION
				{
				mDURATION(); 

				}
				break;
			case 36 :
				// druidG.g:1:279: PERIOD
				{
				mPERIOD(); 

				}
				break;
			case 37 :
				// druidG.g:1:286: INCLUDE
				{
				mINCLUDE(); 

				}
				break;
			case 38 :
				// druidG.g:1:294: WHICH
				{
				mWHICH(); 

				}
				break;
			case 39 :
				// druidG.g:1:300: CONTAINS
				{
				mCONTAINS(); 

				}
				break;
			case 40 :
				// druidG.g:1:309: SORT
				{
				mSORT(); 

				}
				break;
			case 41 :
				// druidG.g:1:314: HINT
				{
				mHINT(); 

				}
				break;
			case 42 :
				// druidG.g:1:319: AS
				{
				mAS(); 

				}
				break;
			case 43 :
				// druidG.g:1:322: FIELD_ACCESS
				{
				mFIELD_ACCESS(); 

				}
				break;
			case 44 :
				// druidG.g:1:335: FROM
				{
				mFROM(); 

				}
				break;
			case 45 :
				// druidG.g:1:340: WHERE
				{
				mWHERE(); 

				}
				break;
			case 46 :
				// druidG.g:1:346: BETWEEN
				{
				mBETWEEN(); 

				}
				break;
			case 47 :
				// druidG.g:1:354: AND
				{
				mAND(); 

				}
				break;
			case 48 :
				// druidG.g:1:358: OR
				{
				mOR(); 

				}
				break;
			case 49 :
				// druidG.g:1:361: NOT
				{
				mNOT(); 

				}
				break;
			case 50 :
				// druidG.g:1:365: GROUP
				{
				mGROUP(); 

				}
				break;
			case 51 :
				// druidG.g:1:371: ASC
				{
				mASC(); 

				}
				break;
			case 52 :
				// druidG.g:1:375: DESC
				{
				mDESC(); 

				}
				break;
			case 53 :
				// druidG.g:1:380: ORDER
				{
				mORDER(); 

				}
				break;
			case 54 :
				// druidG.g:1:386: HAVING
				{
				mHAVING(); 

				}
				break;
			case 55 :
				// druidG.g:1:393: BREAK
				{
				mBREAK(); 

				}
				break;
			case 56 :
				// druidG.g:1:399: BY
				{
				mBY(); 

				}
				break;
			case 57 :
				// druidG.g:1:402: LIMIT
				{
				mLIMIT(); 

				}
				break;
			case 58 :
				// druidG.g:1:408: LIKE
				{
				mLIKE(); 

				}
				break;
			case 59 :
				// druidG.g:1:413: THEN
				{
				mTHEN(); 

				}
				break;
			case 60 :
				// druidG.g:1:418: JAVASCRIPT
				{
				mJAVASCRIPT(); 

				}
				break;
			case 61 :
				// druidG.g:1:429: JOIN
				{
				mJOIN(); 

				}
				break;
			case 62 :
				// druidG.g:1:434: LEFT_JOIN
				{
				mLEFT_JOIN(); 

				}
				break;
			case 63 :
				// druidG.g:1:444: RIGHT_JOIN
				{
				mRIGHT_JOIN(); 

				}
				break;
			case 64 :
				// druidG.g:1:455: ON
				{
				mON(); 

				}
				break;
			case 65 :
				// druidG.g:1:458: OPT_SEMI_COLON
				{
				mOPT_SEMI_COLON(); 

				}
				break;
			case 66 :
				// druidG.g:1:473: OPT_AMPERSAND
				{
				mOPT_AMPERSAND(); 

				}
				break;
			case 67 :
				// druidG.g:1:487: WS
				{
				mWS(); 

				}
				break;
			case 68 :
				// druidG.g:1:490: DATE_YEAR_MONTH_ONLY
				{
				mDATE_YEAR_MONTH_ONLY(); 

				}
				break;
			case 69 :
				// druidG.g:1:511: DATE
				{
				mDATE(); 

				}
				break;
			case 70 :
				// druidG.g:1:516: DATE_HOUR
				{
				mDATE_HOUR(); 

				}
				break;
			case 71 :
				// druidG.g:1:526: DATE_HOUR_MIN
				{
				mDATE_HOUR_MIN(); 

				}
				break;
			case 72 :
				// druidG.g:1:540: DATE_HOUR_MIN_SEC
				{
				mDATE_HOUR_MIN_SEC(); 

				}
				break;
			case 73 :
				// druidG.g:1:558: DATE_HOUR_MIN_SEC_SUB
				{
				mDATE_HOUR_MIN_SEC_SUB(); 

				}
				break;
			case 74 :
				// druidG.g:1:580: DATE_HOUR_MIN_SEC_SUB_TZ
				{
				mDATE_HOUR_MIN_SEC_SUB_TZ(); 

				}
				break;
			case 75 :
				// druidG.g:1:605: DATE_HOUR_MIN_SEC_SUB_UTC_TZ
				{
				mDATE_HOUR_MIN_SEC_SUB_UTC_TZ(); 

				}
				break;
			case 76 :
				// druidG.g:1:634: ARITH_OPER
				{
				mARITH_OPER(); 

				}
				break;
			case 77 :
				// druidG.g:1:645: EQUALS
				{
				mEQUALS(); 

				}
				break;
			case 78 :
				// druidG.g:1:652: COMPARE_OPER
				{
				mCOMPARE_OPER(); 

				}
				break;
			case 79 :
				// druidG.g:1:665: ID
				{
				mID(); 

				}
				break;
			case 80 :
				// druidG.g:1:668: SINGLE_QUOTE_STRING
				{
				mSINGLE_QUOTE_STRING(); 

				}
				break;
			case 81 :
				// druidG.g:1:688: LONG
				{
				mLONG(); 

				}
				break;
			case 82 :
				// druidG.g:1:693: FLOAT
				{
				mFLOAT(); 

				}
				break;

		}
	}
356885
@Override
    public List<QueryRow> reducedQuery(QueryOptions options) throws CouchbaseLiteException {
        final Predicate<QueryRow> postFilter = options.getPostFilter();

        final int groupLevel = options.getGroupLevel();
        final boolean group = options.isGroup() || (groupLevel > 0);
        final Reducer reduce = delegate.getReduce();
        if (options.isReduceSpecified()) {
            if (options.isReduce() && reduce == null) {
                Log.w(TAG, String.format(Locale.ENGLISH,
                        "Cannot use reduce option in view %s which has no reduce block defined",
                        name));
                throw new CouchbaseLiteException(Status.BAD_PARAM);
            }
        }

        final List<Object> keysToReduce = new ArrayList<Object>(REDUCE_BATCH_SIZE);
        final List<Object> valuesToReduce = new ArrayList<Object>(REDUCE_BATCH_SIZE);
        final Object[] lastKeys = new Object[1];
        lastKeys[0] = null;
        final SQLiteViewStore that = this;

        final List<QueryRow> rows = new ArrayList<QueryRow>();

        Status status = runQuery(options, new QueryRowBlock() {
            @Override
            public Status onRow(byte[] keyData, byte[] valueData, String docID, Cursor cursor) {
                JsonDocument keyDoc = new JsonDocument(keyData);
                JsonDocument valueDoc = new JsonDocument(valueData);
                assert (keyDoc != null);
                Object keyObject = keyDoc.jsonObject();
                if (group && !groupTogether(keyObject, lastKeys[0], groupLevel)) {
                    if (lastKeys[0] != null) {
                        // This pair starts a new group, so reduce & record the last one:
                        Object key = groupKey(lastKeys[0], groupLevel);
                        Object reduced = (reduce != null) ?
                                reduce.reduce(keysToReduce, valuesToReduce, false) :
                                null;
                        QueryRow row = new QueryRow(null, 0, key, reduced, null);
                        if (postFilter == null || postFilter.apply(row))
                            rows.add(row);
                        keysToReduce.clear();
                        valuesToReduce.clear();
                    }
                    lastKeys[0] = keyObject;
                }
                //Log.v(TAG, String.format("Query %s: Will reduce row with key=%s, value=%s",
                //        name, new String(keyData), new String(valueData)));

                /*
                Object valueOrData = valueData;
                if(reduce != null && rowValueIsEntireDoc(valueData)){
                    // map fn emitted 'doc' as value, which was stored as a "*" placeholder; expand now:
                    long sequence = Long.valueOf(cursor.getString(3));
                    RevisionInternal rev = store.getDocument(docID, sequence);
                    if(rev == null)
                        Log.w(TAG, String.format("%s: Couldn't load doc for row value", this));
                    else
                        valueOrData = rev.getProperties();
                }
                */
                keysToReduce.add(keyObject);
                valuesToReduce.add(valueDoc.jsonObject());
                return new Status(Status.OK);
            }
        });

        if (keysToReduce != null && keysToReduce.size() > 0) {
            // Finish the last group (or the entire list, if no grouping):
            Object key = group ? groupKey(lastKeys[0], groupLevel) : null;
            Object reduced = (reduce != null) ?
                    reduce.reduce(keysToReduce, valuesToReduce, false) :
                    null;
            Log.v(TAG, String.format(Locale.ENGLISH, "Query %s: Reduced to key=%s, value=%s",
                    name, key, reduced));
            QueryRow row = new QueryRow(null, 0, key, reduced, null);
            if (postFilter == null || postFilter.apply(row)) {
                rows.add(row);
            }
        }
        return rows;
    }
966732
public static Pair<Grouper<RowBasedKey>, Accumulator<AggregateResult, Row>> createGrouperAccumulatorPair(
      final GroupByQuery query,
      final boolean isInputRaw,
      final Map<String, ValueType> rawInputRowSignature,
      final GroupByQueryConfig config,
      final Supplier<ByteBuffer> bufferSupplier,
      final LimitedTemporaryStorage temporaryStorage,
      final ObjectMapper spillMapper,
      final AggregatorFactory[] aggregatorFactories,
      final int mergeBufferSize
  )
  {
    return createGrouperAccumulatorPair(
        query,
        isInputRaw,
        rawInputRowSignature,
        config,
        bufferSupplier,
        null,
        SINGLE_THREAD_CONCURRENCY_HINT,
        temporaryStorage,
        spillMapper,
        aggregatorFactories,
        null,
        UNKNOWN_THREAD_PRIORITY,
        false,
        UNKNOWN_TIMEOUT,
        mergeBufferSize
    );
  }
860318
@Override
	public void prepare() throws Exception {
		final TaskConfig config = this.taskContext.getTaskConfig();
		if (config.getDriverStrategy() != DriverStrategy.ALL_REDUCE) {
			throw new Exception("Unrecognized driver strategy for AllReduce driver: " + config.getDriverStrategy().name());
		}
		
		TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
		this.serializer = serializerFactory.getSerializer();
		this.input = this.taskContext.getInput(0);
	}
990047
private void logOutOfMemoryError()
  {
    logger.error("Dump some crucial information below:\n" +
                 "Total milliseconds waiting for chunks: {},\n" +
                 "Total memory used: {}, Max heap size: {}, total download time: {} millisec,\n" +
                 "total parsing time: {} milliseconds, total chunks: {},\n" +
                 "currentMemoryUsage in Byte: {}, currentMemoryLimit in Bytes: {} \n" +
                 "nextChunkToDownload: {}, nextChunkToConsume: {}\n" +
                 "Several suggestions to try to resolve the OOM issue:\n" +
                 "1. increase the JVM heap size if you have more space; or \n" +
                 "2. use CLIENT_MEMORY_LIMIT to reduce the memory usage by the JDBC driver " +
                 "(https://docs.snowflake.net/manuals/sql-reference/parameters.html#client-memory-limit)" +
                 "3. please make sure 2 * CLIENT_PREFETCH_THREADS * CLIENT_RESULT_CHUNK_SIZE < CLIENT_MEMORY_LIMIT. " +
                 "If not, please reduce CLIENT_PREFETCH_THREADS and CLIENT_RESULT_CHUNK_SIZE too.",
                 numberMillisWaitingForChunks,
                 Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory(),
                 totalMillisDownloadingChunks.get(),
                 totalMillisParsingChunks.get(), chunks.size(), currentMemoryUsage, memoryLimit,
                 nextChunkToDownload, nextChunkToConsume);
  }
1008958
@Override
	public void setup(AbstractInvokable parent) {
		this.parent = parent;

		@SuppressWarnings("unchecked")
		final GroupReduceFunction<IN, OUT> combiner =
			BatchTask.instantiateUserCode(this.config, userCodeClassLoader, GroupReduceFunction.class);
		this.reducer = combiner;
		FunctionUtils.setFunctionRuntimeContext(combiner, getUdfRuntimeContext());
	}