Pull the LTI POST data into our own data structure

We follow our naming conventions that match the column names in
our lti_ tables.
490510
public static function extractPost($needed=self::ALL, $input=false) {
        // Unescape each time we use this stuff - someday we won't need this...
        $needed = self::patchNeeded($needed);
        if ( $input === false ) $input = $_POST;
        $FIXED = array();
        foreach($input as $key => $value ) {
            if (get_magic_quotes_gpc()) $value = stripslashes($value);
            if ( strpos($key, "custom_") === 0 ) {
                $newkey = substr($key,7);
                // Need to deal with custom_context_id=$Context.id
                if ( strpos($value,"$") === 0 ) {
                    $short_value = strtolower(substr($value,1));
                    $short_value = str_replace('.','_',$short_value);
                    if ( $newkey == $short_value ) {
                        continue;
                    }
                }
                if ( !isset($FIXED[$newkey]) ) $FIXED[$newkey] = $value;
            }
            $FIXED[$key] = $value;
        }
        $retval = array();
        $retval['key'] = isset($FIXED['oauth_consumer_key']) ? $FIXED['oauth_consumer_key'] : null;
        $retval['nonce'] = isset($FIXED['oauth_nonce']) ? $FIXED['oauth_nonce'] : null;
        $link_id = isset($FIXED['resource_link_id']) ? $FIXED['resource_link_id'] : null;
        $link_id = isset($FIXED['custom_resource_link_id']) ? $FIXED['custom_resource_link_id'] : $link_id;
        $retval['link_id'] = $link_id;

        $user_id = isset($FIXED['person_sourcedid']) ? $FIXED['person_sourcedid'] : null;
        $user_id = isset($FIXED['user_id']) ? $FIXED['user_id'] : $user_id;
        $user_id = isset($FIXED['custom_user_id']) ? $FIXED['custom_user_id'] : $user_id;
        $retval['user_id'] = $user_id;

        $context_id = isset($FIXED['courseoffering_sourcedid']) ? $FIXED['courseoffering_sourcedid'] : null;
        $context_id = isset($FIXED['context_id']) ? $FIXED['context_id'] : $context_id;
        $context_id = isset($FIXED['custom_context_id']) ? $FIXED['custom_context_id'] : $context_id;
        $retval['context_id'] = $context_id;

        // Sanity checks
        if ( ! $retval['key'] ) return "Missing oauth_consumer_key";
        if ( ! $retval['nonce'] ) return "Missing nonce";
        if ( in_array(self::USER, $needed) && ! $retval['user_id'] ) return "Missing required user_id";
        if ( in_array(self::CONTEXT, $needed) && ! $retval['context_id'] ) return "Missing required context_id";
        if ( in_array(self::LINK, $needed) && ! $retval['link_id'] ) return "Missing required resource_link_id";

        // LTI 1.x settings and Outcomes
        $retval['service'] = isset($FIXED['lis_outcome_service_url']) ? $FIXED['lis_outcome_service_url'] : null;
        $retval['sourcedid'] = isset($FIXED['lis_result_sourcedid']) ? $FIXED['lis_result_sourcedid'] : null;

        // LTI 2.x settings and Outcomes
        $retval['result_url'] = isset($FIXED['custom_result_url']) ? $FIXED['custom_result_url'] : null;
        $retval['link_settings_url'] = isset($FIXED['custom_link_settings_url']) ? $FIXED['custom_link_settings_url'] : null;
        $retval['context_settings_url'] = isset($FIXED['custom_context_settings_url']) ? $FIXED['custom_context_settings_url'] : null;

        // LTI 1.x / 2.x Service endpoints
        $retval['ext_memberships_id'] = isset($FIXED['ext_memberships_id']) ? $FIXED['ext_memberships_id'] : null;
        $retval['ext_memberships_url'] = isset($FIXED['ext_memberships_url']) ? $FIXED['ext_memberships_url'] : null;
        $retval['lineitems_url'] = isset($FIXED['lineitems_url']) ? $FIXED['lineitems_url'] : null;
        $retval['memberships_url'] = isset($FIXED['memberships_url']) ? $FIXED['memberships_url'] : null;

        // Context
        $retval['context_title'] = isset($FIXED['context_title']) ? $FIXED['context_title'] : null;
        $retval['link_title'] = isset($FIXED['resource_link_title']) ? $FIXED['resource_link_title'] : null;

        $retval['user_locale'] = isset($FIXED['launch_presentation_locale']) ? $FIXED['launch_presentation_locale'] : null;

        // Getting email from LTI 1.x and LTI 2.x
        $retval['user_email'] = isset($FIXED['lis_person_contact_email_primary']) ? $FIXED['lis_person_contact_email_primary'] : null;
        $retval['user_email'] = isset($FIXED['custom_person_email_primary']) ? $FIXED['custom_person_email_primary'] : $retval['user_email'];

        $retval['user_image'] = isset($FIXED['user_image']) ? $FIXED['user_image'] : null;

        // Displayname from LTI 2.x
        if ( isset($FIXED['person_name_full']) ) {
            $retval['user_displayname'] = $FIXED['custom_person_name_full'];
        } else if ( isset($FIXED['custom_person_name_given']) && isset($FIXED['custom_person_name_family']) ) {
            $retval['user_displayname'] = $FIXED['custom_person_name_given'].' '.$FIXED['custom_person_name_family'];
        } else if ( isset($FIXED['custom_person_name_given']) ) {
            $retval['user_displayname'] = $FIXED['custom_person_name_given'];
        } else if ( isset($FIXED['custom_person_name_family']) ) {
            $retval['user_displayname'] = $FIXED['custom_person_name_family'];

        // Displayname from LTI 1.x
        } else if ( isset($FIXED['lis_person_name_full']) ) {
            $retval['user_displayname'] = $FIXED['lis_person_name_full'];
        } else if ( isset($FIXED['lis_person_name_given']) && isset($FIXED['lis_person_name_family']) ) {
            $retval['user_displayname'] = $FIXED['lis_person_name_given'].' '.$FIXED['lis_person_name_family'];
        } else if ( isset($FIXED['lis_person_name_given']) ) {
            $retval['user_displayname'] = $FIXED['lis_person_name_given'];
        } else if ( isset($FIXED['lis_person_name_family']) ) {
            $retval['user_displayname'] = $FIXED['lis_person_name_family'];
        }

        // Trim out repeated spaces and/or weird whitespace from the user_displayname
        if ( isset($retval['user_displayname']) ) {
            $retval['user_displayname'] = trim(preg_replace('/\s+/', ' ',$retval['user_displayname']));
        }

        // Get the role
        $retval['role'] = self::ROLE_LEARNER;
        $roles = '';
        if ( isset($FIXED['custom_membership_role']) ) { // From LTI 2.x
            $roles = $FIXED['custom_membership_role'];
        } else if ( isset($FIXED['roles']) ) { // From LTI 1.x
            $roles = $FIXED['roles'];
        }

        if ( strlen($roles) > 0 ) {
            $roles = strtolower($roles);
            if ( ! ( strpos($roles,'instructor') === false ) ) $retval['role'] = self::ROLE_INSTRUCTOR;
            if ( ! ( strpos($roles,'administrator') === false ) ) $retval['role'] = self::ROLE_ADMINISTRATOR;
            // Local superuser would be 10000
        }

        // Copy in some extensions.  Backwards compatibility for canvas xapi urls in legacy cartridges
        $sub_canvas_caliper_url = U::get($FIXED,'sub_canvas_xapi_url');
        if ( $sub_canvas_caliper_url ) {
            $sub_canvas_caliper_url = str_replace('xapi','caliper',$sub_canvas_caliper_url);
        }
        $sub_canvas_caliper_url = U::get($FIXED,'sub_canvas_caliper_url', $sub_canvas_caliper_url);
        if ($sub_canvas_caliper_url ) $retval['sub_canvas_caliper_url'] = $sub_canvas_caliper_url;

        $sub_caliper_url = U::get($FIXED,'sub_caliper_url');
        if ($sub_caliper_url ) $retval['sub_caliper_url'] = $sub_caliper_url;

        return $retval;
    }
790623
protected function add_data_for_post($post) {
        if (!isset($this->adhocdata[$post->course])) {
            $this->adhocdata[$post->course] = [];
        }

        if (!isset($this->adhocdata[$post->course][$post->forum])) {
            $this->adhocdata[$post->course][$post->forum] = [];
        }

        if (!isset($this->adhocdata[$post->course][$post->forum][$post->discussion])) {
            $this->adhocdata[$post->course][$post->forum][$post->discussion] = [];
        }

        $this->adhocdata[$post->course][$post->forum][$post->discussion][$post->id] = $post->id;
    }
765118
def to_api_data(self, restrict_keys=None):
        
        cc = self._cc  # alias
        data = {
            cc('column_width'): self._column_width,
            cc('horizontal_alignment'): self._horizontal_alignment,
            cc('row_height'): self._row_height,
            cc('vertical_alignment'): self._vertical_alignment,
            cc('wrap_text'): self._wrap_text,
        }

        if restrict_keys:
            for key in list(data.keys()):
                if key not in restrict_keys:
                    del data[key]
        return data
702492
protected function doPostGetData()
    {
        foreach ($this->dataTransformers as $dataTransformer) {
            if ($dataTransformer instanceof PostGetDataTransformerInterface) {
                $dataTransformer->postGetData($this->config);
            }
        }
    }
1055322
def process_data(self, text, colsep=u"\t", rowsep=u"\n",
                     transpose=False, skiprows=0, comments='#'):
        """"""
        data = self._shape_text(text, colsep, rowsep, transpose, skiprows,
                                comments)
        self._model = PreviewTableModel(data)
        self.setModel(self._model)
670140
def to_table_data(self):
        

        self._validate_source_data()

        for table_key, json_records in six.iteritems(self._buffer):
            self._loader.inc_table_count()
            self._table_key = table_key

            yield TableData(
                self._make_table_name(),
                ["key", "value"],
                [record for record in json_records.items()],
                dp_extractor=self._loader.dp_extractor,
                type_hints=self._extract_type_hints(),
            )
1199373
def import_data(self, data_items):
        
        # keep counts of all actions
        record = {
            'insert': 0, 'update': 0, 'noop': 0,
            'start': utcnow(),
            'records': {
                'insert': [],
                'update': [],
                'noop': [],
            }
        }

        for json_id, data in self._prepare_imports(data_items):
            obj_id, what = self.import_item(data)
            self.json_to_db_id[json_id] = obj_id
            record['records'][what].append(obj_id)
            record[what] += 1

        # all objects are loaded, a perfect time to do inter-object resolution and other tasks
        self.postimport()

        record['end'] = utcnow()

        return {self._type: record}
345286
def _setup_from_data(self, data):
        

        fitted_state = {}

        _raise_error_if_not_of_type(data, [_SFrame])

        feature_columns = _internal_utils.get_column_names(data, self._exclude, self._features)

        if not feature_columns:
            raise RuntimeError("No valid feature columns specified in transformation.")

        fitted_state["features"] = feature_columns

        ################################################################################
        # Helper functions

        def get_valid_interpretations():
            return list(n.split("__")[0] for n in dir(_interpretations) if not n.startswith("_"))

        ################################################################################
        # Check input data.

        if not isinstance(data, _SFrame):
            raise TypeError("`data` parameter must be an SFrame.")

        all_col_names = set(feature_columns)

        column_interpretations = self._get("column_interpretations").copy()

        # Make sure all the interpretations are valid.
        for k, v in column_interpretations.items():
            if k not in all_col_names:
                raise ValueError("Column '%s' in column_interpretations, but not found in `data`." % k)

        # Get the automatic column interpretations.
        for col_name in feature_columns:
            if col_name not in column_interpretations:
                n = column_interpretations[col_name] = infer_column_interpretation(data[col_name])

                if n.startswith("unknown"):
                    raise ValueError("Interpretation inference failed on column '%s'; %s"
                                     % (col_name, n[len("unknown"):].strip()))

        # Now, build up the feature transforms.
        transforms = {}
        input_types = {}

        output_column_prefix = self._get("output_column_prefix")

        assert output_column_prefix is None or type(output_column_prefix) is str

        tr_chain = []
        for col_name in feature_columns:
            in_type = input_types[col_name] = data[col_name].dtype

            intr_func = _get_interpretation_function(column_interpretations[col_name], in_type)
            tr_list = intr_func(col_name, output_column_prefix)
            transforms[col_name] = tr_list
            tr_chain += tr_list

        fitted_state["transform_chain"] = _TransformerChain(tr_chain)
        fitted_state["transforms"] = transforms
        fitted_state["input_types"] = input_types

        fitted_state["column_interpretations"] = column_interpretations

        self.__proxy__.update(fitted_state)
256701
def parse_data_to_internal(self, include_custom=False):
        
        for element in self.data:
            self.data[element].parse_data_to_internal()
        if include_custom:
            for element in self.custom_data:
                self.custom_data[element].parse_data_to_internal()
502515
def name_columns(self, sub):
        
        if self.column_id is not None and len(sub) > self.column_id:
            self.split_name = sub[self.column_id]
        for split in self.surrogates:
            split.name_columns(sub)
125972
def open_data(self, text, colsep=u"\t", rowsep=u"\n",
                  transpose=False, skiprows=0, comments='#'):
        """"""
        if pd:
            self.pd_text = text
            self.pd_info = dict(sep=colsep, lineterminator=rowsep,
                skiprows=skiprows, comment=comments)
            if colsep is None:
                self.pd_info = dict(lineterminator=rowsep, skiprows=skiprows,
                    comment=comments, delim_whitespace=True)
        self._table_view.process_data(text, colsep, rowsep, transpose,
                                      skiprows, comments)
1013061
def results_to_table(self):
        """"""
        # Get Labels
        baseline_labels = self.nearest_targets['origin_id'].values
        mcf_labels = self.results['model_labels']['labels'].values

        # Create the outcomes
        outcome = pd.DataFrame({
                'the_geom': [
                    'SRID=4326;Point({lng} {lat})'.format(lng=v[0], lat=v[1])
                    for v in zip(self.results['model_labels']['lng'].values, self.results['model_labels']['lat'].values)],
                'target_lng': self.results['model_labels']['lng'].values,
                'target_lng': self.results['model_labels']['lat'].values,
                'origin_lng': self.origins.reindex(baseline_labels)['lng'].values,
                'origin_lat': self.origins.reindex(baseline_labels)['lat'].values,
                'target_id': self.results['model_labels'].target_id,
                'sales': self.results['model_labels'][self.demand_col].values,
                'labels': baseline_labels
            }
        )

        outcomes2 = pd.DataFrame({
                'the_geom': [
                    'SRID=4326;Point({lng} {lat})'.format(lng=v[0], lat=v[1])
                    for v in zip(
                        self.results['model_labels']['lng'].values,
                        self.results['model_labels']['lat'].values
                    )
                ],
                'target_lng': self.results['model_labels']['lng'].values,
                'target_lat': self.results['model_labels']['lat'].values,
                'origin_lng': self.origins.reindex(mcf_labels)['lng'].values,
                'origin_lat': self.origins.reindex(mcf_labels)['lat'].values,
                'target_id': self.results['model_labels'].target_id,
                'sales': self.results['model_labels'][self.demand_col].values,
                'labels': mcf_labels
            },
            index=self.results['model_labels'].target_id
        )
        now = datetime.datetime.now()
        out_table = 'mincostflow_{}'.format(now.strftime("%Y_%m_%d_%H_%M_%S"))
        logging.info('Writing output to {}'.format(out_table))
        self.context.write(outcomes2.reset_index(drop=True), out_table)
        logging.info('Table {} written to CARTO'.format(out_table))
        return out_table
256012
def set_column_names(self, column_names):
        
        if len(self._rows):
            raise NotImplementedError("You can't manually set columns once data has been added")

        self._update_internal_column_state(column_names)
1214377
def _loadData(self, data):
        
        self._data = data
        self.id = cast(int, data.attrib.get('id'))
        self.role = data.attrib.get('role')
        self.tag = data.attrib.get('tag')
        # additional attributes only from hub search
        self.key = data.attrib.get('key')
        self.librarySectionID = cast(int, data.attrib.get('librarySectionID'))
        self.librarySectionTitle = data.attrib.get('librarySectionTitle')
        self.librarySectionType = data.attrib.get('librarySectionType')
        self.tagType = cast(int, data.attrib.get('tagType'))
        self.thumb = data.attrib.get('thumb')
721446
def process_subscription_data(post_data):
    """"""
    subscription_data = post_data.pop("subscription", {})
    # As our database saves the auth and p256dh key in separate field,
    # we need to refactor it and insert the auth and p256dh keys in the same dictionary
    keys = subscription_data.pop("keys", {})
    subscription_data.update(keys)
    # Insert the browser name
    subscription_data["browser"] = post_data.pop("browser")
    return subscription_data
1344417
protected function pullPost() {
		if (!is_array($this->postVars)) {
			$this->postVars = $this->request->getParsedBody();
			
			if (!is_array($this->postVars)) {
				$this->postVars = [];
			}
		}
		
		return $this->postVars;
	}
887087
def populate_from_staging(self, staging_table, from_column_list, output_table):
        
        self.sql_text += 'INSERT INTO ' + output_table + ' (\n'
        for c in self.col_list:
            if c != '':
                self.sql_text += '    ' + c + ',\n'
        self.sql_text += '    ' + self.date_updated_col + ') (\n'
        self.sql_text += '    SELECT \n'
        for c in from_column_list:
            if c != '':
                self.sql_text += '        ' + c + ',\n'
        self.sql_text += '        SYSDATE \n    FROM ' + staging_table
        self.sql_text += '\n);\n'