Take a string of form data as CSV and convert to insert statements, return template and data values
420480
def processFormData(self, data, dataset_name):
        """"""

        # Get the cols for this dataset
        cols = self.datasets[dataset_name]
        reader = self.getCSVReader(data, reader_type=csv.reader)

        # Get fieldnames from first line of reader, what is left is set of rows
        fieldnames = next(reader)

        # Check columns
        for col in cols:
            varname = col["varname"]
            if varname not in fieldnames:
                raise ValueError("Column %s not found in data for dataset %s" % (varname, dataset_name,))

        # Now call overrideen methods of base classes to process this data
        self._processDML(dataset_name, cols, reader)
1329529
def build_insert(table_name, attributes):
    
    sql = "INSERT INTO %s" %(table_name)
    column_str = u""
    value_str = u""
    for index, (key, value) in enumerate(attributes.items()):
        if index > 0:
            column_str += u","
            value_str += u","
        column_str += key
        value_str += value_to_sql_str(value)
    sql = sql + u"(%s) VALUES(%s)" %(column_str, value_str)
    return sql
393266
def get_template(filename_or_string, is_string=False):
    '''
    
    '''

    # Cache against string sha or just the filename
    cache_key = sha1_hash(filename_or_string) if is_string else filename_or_string

    if cache_key in TEMPLATES:
        return TEMPLATES[cache_key]

    if is_string:
        # Set the input string as our template
        template_string = filename_or_string

    else:
        # Load template data into memory
        with open(filename_or_string, 'r') as file_io:
            template_string = file_io.read()

    TEMPLATES[cache_key] = Template(template_string, keep_trailing_newline=True)
    return TEMPLATES[cache_key]
889403
def csv_to_sqlite(
    input_filename,
    output_filename,
    samples=None,
    dialect=None,
    batch_size=10000,
    encoding="utf-8",
    callback=None,
    force_types=None,
    chunk_size=8388608,
    table_name="table1",
    schema=None,
):
    ""

    # TODO: automatically detect encoding if encoding == `None`
    # TODO: should be able to specify fields
    # TODO: if table_name is "2019" the final name will be "field_2019" - must
    #       be "table_2019"
    # TODO: if schema is provided and the names are in uppercase, this function
    #       will fail

    if dialect is None:  # Get a sample to detect dialect
        fobj = open_compressed(input_filename, mode="rb")
        sample = fobj.read(chunk_size)
        dialect = rows.plugins.csv.discover_dialect(sample, encoding=encoding)
    elif isinstance(dialect, six.text_type):
        dialect = csv.get_dialect(dialect)

    if schema is None:  # Identify data types
        fobj = open_compressed(input_filename, encoding=encoding)
        data = list(islice(csv.DictReader(fobj, dialect=dialect), samples))
        schema = rows.import_from_dicts(data).fields
        if force_types is not None:
            schema.update(force_types)

    # Create lazy table object to be converted
    # TODO: this lazyness feature will be incorported into the library soon so
    #       we can call here `rows.import_from_csv` instead of `csv.reader`.
    reader = csv.reader(
        open_compressed(input_filename, encoding=encoding), dialect=dialect
    )
    header = make_header(next(reader))  # skip header
    table = rows.Table(fields=OrderedDict([(field, schema[field]) for field in header]))
    table._rows = reader

    # Export to SQLite
    return rows.export_to_sqlite(
        table,
        output_filename,
        table_name=table_name,
        batch_size=batch_size,
        callback=callback,
    )
571866
def process_formdata(self, valuelist):
        ''''''
        super(EmptyNone, self).process_formdata(valuelist)
        self.data = self.data or None
387730
def template(basedir, text, vars, lookup_fatal=True, expand_lists=False):
    '''  '''

    try:
        text = text.decode('utf-8')
    except UnicodeEncodeError:
        pass # already unicode
    text = varReplace(basedir, unicode(text), vars, lookup_fatal=lookup_fatal, expand_lists=expand_lists)
    return text
447203
function getFormData(form) {
    var elements = form.elements;

    var fields = Object.keys(elements).filter(function(k) {
          return (elements[k].name !== "honeypot");
    }).map(function(k) {
      if(elements[k].name !== undefined) {
        return elements[k].name;
      // special case for Edge's html collection
      }else if(elements[k].length > 0){
        return elements[k].item(0).name;
      }
    }).filter(function(item, pos, self) {
      return self.indexOf(item) == pos && item;
    });

    var formData = {};
    fields.forEach(function(name){
      var element = elements[name];
      
      // singular form elements just have one value
      formData[name] = element.value;

      // when our element has multiple items, get their values
      if (element.length) {
        var data = [];
        for (var i = 0; i < element.length; i++) {
          var item = element.item(i);
          if (item.checked || item.selected) {
            data.push(item.value);
          }
        }
        formData[name] = data.join(', ');
      }
    });

    // add form-specific values into the data
    formData.formDataNameOrder = JSON.stringify(fields);
    formData.formGoogleSheetName = form.dataset.sheet || "responses"; // default sheet name
    formData.formGoogleSendEmail = form.dataset.email || ""; // no email by default

    console.log(formData);
    return formData;
  }
1234276
function processString(state, value, data) {
  return value.replace(TEMPLATE_REGION, function(path) {
    var insert = envEval(state, path.slice(2, -1), data, value);
    return state.options.blankNullUndefined && insert == null ? '' : insert;
  });
}
901132
def make_csv(data):
    
    # Convert sub-dicts to news cols
    for entry in data:
        rval = entry
        if isinstance(data, dict):
            rval = data[entry]
        todel = []
        toadd = {}
        for key, val in rval.items():
            if isinstance(val, dict):
                for key2, val2 in val.items():
                    toadd[str(key) + "[" + str(key2) + "]"] = val2
                todel.append(key)
        for k in todel:
            del rval[k]
        for k, v in toadd.items():
            rval[k] = v

    # Convert everything to CSV
    columns = set()
    output = [[]]
    if isinstance(data, dict):
        output[0].append("id")
        for entry in data:
            for col in data[entry]:
                columns.add(col)
    else:
        for entry in data:
            for col in entry:
                columns.add(col)

    columns = sorted(columns)

    for col in columns:
        output[0].append(col)

    if isinstance(data, dict):
        for entry in data:
            new_output = [str(entry)]
            for col in columns:
                new_output.append(str(data[entry][col]) if col in data[entry] else "")
            output.append(new_output)
    else:
        for entry in data:
            new_output = []
            for col in columns:
                new_output.append(str(entry[col]) if col in entry else "")
            output.append(new_output)

    csv_string = io.StringIO()
    csv_writer = UnicodeWriter(csv_string)
    for row in output:
        csv_writer.writerow(row)
    csv_string.seek(0)
    web.header('Content-Type', 'text/csv; charset=utf-8')
    web.header('Content-disposition', 'attachment; filename=export.csv')
    return csv_string.read()
308182
def format(input, **params):
    
    PARAM_FORMAT_STRING = 'format.string'
    PARAM_FORMAT_INPUT = 'format.input'
    PARAM_RESULT_FIELD = 'result.field'
    IN_DESC_FIELD = 'field'
    IN_DESC_TYPE = 'type'

    format_string = params.get(PARAM_FORMAT_STRING)
    format_inputs = params.get(PARAM_FORMAT_INPUT)
    result_field = params.get(PARAM_RESULT_FIELD)
    for row in input:
        row_input = []
        for desc in format_inputs:
            row_input.append(Converter.convert(row[desc[IN_DESC_FIELD]], desc[IN_DESC_TYPE]))
        row[result_field] = format_string.format(*row_input)
    return input
1157765
function csv(arg) {
  return ( _.isArray(arg) ? arg :
           _.isObject(arg) ? _.values(arg) :
           [arg]).join(', ');
}
1342513
def convert(input_file_name, **kwargs):
    """"""
    delimiter = kwargs["delimiter"] or ","
    quotechar = kwargs["quotechar"] or "|"

    if six.PY2:
        delimiter = delimiter.encode("utf-8")
        quotechar = quotechar.encode("utf-8")

    # Read CSV and form a header and rows list
    with open(input_file_name, "rb") as input_file:
        reader = csv.reader(input_file,
                            encoding="utf-8",
                            delimiter=delimiter,
                            quotechar=quotechar)

        csv_headers = []
        if not kwargs.get("no_header"):
            # Read header from first line
            csv_headers = next(reader)

        csv_rows = [row for row in reader if row]

        # Set default column name if header is not present
        if not csv_headers and len(csv_rows) > 0:
            end = len(csv_rows[0]) + 1
            csv_headers = ["Column {}".format(n) for n in range(1, end)]

    # Render csv to HTML
    html = render_template(csv_headers, csv_rows, **kwargs)

    # Freeze all JS files in template
    return freeze_js(html)
527469
function parseForms()
	{
		preg_match_all('/<form.*>.+<\/form>/isU', $this->html_data, $forms);
		foreach ($forms[0] as $form) {
			$this->button_counter = 0;

			//form details
			preg_match('/<form.*?name=["\']?([\w\s-]*)["\']?[\s>]/is', $form, $form_name);
			if ($form_name) {
				$this->_return[$this->_counter]['form_data']['name'] = preg_replace('/["\'<>]/', '', $form_name[1]);
			}
			preg_match('/<form.*?action=("([^"]*)"|\'([^\']*)\'|[^>\s]*)([^>]*)?>/is', $form, $action);
			if ($action) {
				$this->_return[$this->_counter]['form_data']['action'] = preg_replace('/["\'<>]/', '', $action[1]);
			}
			preg_match('/<form.*?method=["\']?([\w\s]*)["\']?[\s>]/is', $form, $method);
			if ($method) {
				$this->_return[$this->_counter]['form_data']['method'] = preg_replace('/["\'<>]/', '', $method[1]);
			}
			preg_match('/<form.*?enctype=("([^"]*)"|\'([^\']*)\'|[^>\s]*)([^>]*)?>/is', $form, $enctype);
			if ($enctype) {
				$this->_return[$this->_counter]['form_data']['enctype'] = preg_replace('/["\'<>]/', '', $enctype[1]);
			}
			preg_match('/<form.*?id=["\']?([\w\s-]*)["\']?[\s>]/is', $form, $id);
			if ($id) {
				$this->_return[$this->_counter]['form_data']['id'] = preg_replace('/["\'<>]/', '', $id[1]);
			}

			// form elements: input type = hidden
			if (preg_match_all('/<input[^<>]+type=["\']hidden["\'][^<>]*>/isU', $form, $hiddens)) {
				foreach ($hiddens[0] as $hidden) {
					$this->_return[$this->_counter]['form_elements'][$this->_getName($hidden)] = array(
						'type'  =>  'hidden',
						'value'  =>  $this->_getValue($hidden)
					);
				}
			}

			// form elements: input type = text
			if (preg_match_all('/<input[^<>]+type=["\']text["\'][^<>]*>/isU', $form, $texts)) {
				foreach ($texts[0] as $text) {
					$this->_return[$this->_counter]['form_elements'][$this->_getName($text)] = array(
						'type' => 'text',
						'value' => $this->_getValue($text),
						'id' => $this->_getId($text),
						'class' => $this->_getClass($text)
					);
				}
			}

			/**
			 * HJW 18-7-2014 Magento 1.9.01. uses email as type for user credential
			 * form elements: input type = email
			 */
			if (preg_match_all('/<input[^<>]+type=["\']email["\'][^<>]*>/isU', $form, $texts)) {
				foreach ($texts[0] as $text) {
					$this->_return[$this->_counter]['form_elements'][$this->_getName($text)] = array(
						'type'  => 'email',
						'value'  =>  $this->_getValue($text),
						'id'  =>  $this->_getId($text),
						'class'  =>  $this->_getClass($text)
					);
				}
			}

            // form elements: input type = email
            if (preg_match_all('/<input[^<>]+type=["\']email["\'][^<>]*>/isU', $form, $texts)) {
                foreach ($texts[0] as $text) {
                    $this->_return[$this->_counter]['form_elements'][$this->_getName($text)] = array(
                        'type'  => 'email',
                        'value'  =>  $this->_getValue($text),
                        'id'  =>  $this->_getId($text),
                        'class'  =>  $this->_getClass($text)
                    );
                }
            }

			// form elements: input type = password
			if (preg_match_all('/<input[^<>]+type=["\']password["\'][^<>]*>/isU', $form, $passwords)) {
				foreach ($passwords[0] as $password) {
					$this->_return[$this->_counter]['form_elements'][$this->_getName($password)] = array(
						'type'  =>  'password',
						'value'  =>  $this->_getValue($password)
					);
				}
			}

			// form elements: textarea
			if (preg_match_all('/<textarea.*>.*<\/textarea>/isU', $form, $textareas)) {
				foreach ($textareas[0] as $textarea) {
					preg_match('/<textarea.*>(.*)<\/textarea>/isU', $textarea, $textarea_value);
					$this->_return[$this->_counter]['form_elements'][$this->_getName($textarea)] = array(
						'type'  =>  'textarea',
						'value'  =>  $textarea_value[1]
					);
				}
			}

			// form elements: input type = checkbox
			if (preg_match_all('/<input[^<>]+type=["\']checkbox["\'][^<>]*>/isU', $form, $checkboxes)) {
				foreach ($checkboxes[0] as $checkbox) {
					if (preg_match('/checked/is', $checkbox)) {
						$this->_return[$this->_counter]['form_elements'][$this->_getName($checkbox)] = array(
							'type'  =>  'checkbox',
							'value'  =>  'on'
						);
					} else {
						$this->_return[$this->_counter]['form_elements'][$this->_getName($checkbox)] = array(
							'type'  =>  'checkbox',
							'value'  =>  ''
						);
					}
				}
			}

			// form elements: input type = radio
			if (preg_match_all('/<input[^<>]+type=["\']radio["\'][^<>]*>/isU', $form, $radios)) {
				foreach ($radios[0] as $radio) {
					if (preg_match('/checked/i', $radio)) {
						$this->_return[$this->_counter]['form_elements'][$this->_getName($radio)] = array(
							'type'  =>  'radio',
							'value'  =>  $this->_getValue($radio)
						);
					}
				}
			}

			// form elements: input type = submit
			if (preg_match_all('/<input[^<>]+type=["\']submit["\'][^<>]*>/isU', $form, $submits)) {
				foreach ($submits[0] as $submit) {
					$this->_return[$this->_counter]['buttons'][$this->button_counter] = array(
						'type'  => 'submit',
						'name'  => $this->_getName($submit),
						'value'  => $this->_getValue($submit)
					);
					$this->button_counter++;
				}
			}

			// form elements: input type = button
			if (preg_match_all('/<input[^<>]+type=["\']button["\'][^<>]*>/isU', $form, $buttons)) {
				foreach ($buttons[0] as $button) {
					$this->_return[$this->_counter]['buttons'][$this->button_counter] = array(
						'type'  => 'button',
						'name'  => $this->_getName($button),
						'value'  => $this->_getValue($button)
					);
					$this->button_counter++;
				}
			}

			// form elements: input type = reset
			if (preg_match_all('/<input[^<>]+type=["\']reset["\'][^<>]*>/isU', $form, $resets)) {
				foreach ($resets[0] as $reset) {
					$this->_return[$this->_counter]['buttons'][$this->button_counter] = array(
						'type'  => 'reset',
						'name'  => $this->_getName($reset),
						'value'  => $this->_getValue($reset)
					);
					$this->button_counter++;
				}
			}

			// form elements: input type = image
			if (preg_match_all('/<input[^<>]+type=["\']image["\'][^<>]*>/isU', $form, $images)) {
				foreach ($images[0] as $image) {
					$this->_return[$this->_counter]['buttons'][$this->button_counter] = array(
						'type'  => 'image',
						'name'  => $this->_getName($image),
						'value'  => $this->_getValue($image)
					);
					$this->button_counter++;
				}
			}

			// input type=select entries
			// Here I have to go on step around to grep at first all select names and then
			// the content. Seems not to work in an other way
			if (preg_match_all('/<select.*>.+<\/select>/isU', $form, $selects)) {
				foreach ($selects[0] as $select) {
					if (preg_match_all('/<option.*>.+<\/option>/isU', $select, $all_options)) {
						$option_value = '';
						foreach ($all_options[0] as $option) {
							if (preg_match('/selected/i', $option)) {
								if (preg_match('/value=["\'](.*)["\']\s/isU', $option, $option_value)) {
									$option_value = $option_value[1];
									$found_selected = 1;
								} else {
									preg_match('/<option.*>(.*)<\/option>/isU', $option, $option_value);
									$option_value = $option_value[1];
									$found_selected = 1;
								}
							}
						}
						if (!isset($found_selected)) {
							if (preg_match('/value=["\'](.*)["\']/isU', $all_options[0][0], $option_value)) {
								$option_value = $option_value[1];
							} else {
								preg_match('/<option>(.*)<\/option>/isU', $all_options[0][0], $option_value);
								$option_value = $option_value[1];
							}
						} else {
							unset($found_selected);
						}
						$this->_return[$this->_counter]['form_elements'][$this->_getName($select)] = array(
							'type'  => 'select',
							'value'  => trim($option_value)
						);
					}
				}
			}

			# form elements: input type = --not defined--
			if ( preg_match_all('/<input[^<>]+name=["\'](.*)["\'][^<>]*>/isU', $form, $inputs)) {
				foreach ($inputs[0] as $input) {
					if ( !preg_match('/type=("([^"]*)"|\'([^\']*)\'|[^>\s]*)([^>]*)?>/is', $input) ) {
						if ( !isset($this->_return[$this->_counter]['form_elements'][$this->_getName($input)]) ) {
							$this->_return[$this->_counter]['form_elements'][$this->_getName($input)] =
								array(
									'type'  => 'text',
									'value'  =>  $this->_getValue($input),
									'id'  =>  $this->_getId($input),
									'class'  =>  $this->_getClass($input)
								);

						}
					}
				}
			}

			// Update the form counter if we have more then 1 form in the HTML table
			$this->_counter++;
		}

		return $this->_return;
	}
1055194
def compile_insert(self, query, values):
        
        table = self.wrap_table(query.from__)

        if not isinstance(values, list):
            values = [values]

        # If there is only one row to insert, we just use the normal grammar
        if len(values) == 1:
            return super(SQLiteQueryGrammar, self).compile_insert(query, values)

        names = self.columnize(values[0].keys())

        columns = []

        # SQLite requires us to build the multi-row insert as a listing of select with
        # unions joining them together. So we'll build out this list of columns and
        # then join them all together with select unions to complete the queries.
        for column in values[0].keys():
            columns.append("%s AS %s" % (self.get_marker(), self.wrap(column)))

        columns = [", ".join(columns)] * len(values)

        return "INSERT INTO %s (%s) SELECT %s" % (
            table,
            names,
            " UNION ALL SELECT ".join(columns),
        )
1043831
def rectify_multipart_form_data(self, data):
        
        for name, field in self.base_fields.items():
            try:
                field.implode_multi_values(name, data)
            except AttributeError:
                pass
        return data
315342
def encode_multipart_formdata(fields):
    
    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
    CRLF = '\r\n'.encode('utf-8')
    L = []
    for (key, value) in fields:
        if not isinstance(value, types.InputFile):
            _append_bytes(L, '--' + BOUNDARY)
            _append_bytes(L, 'Content-Disposition: form-data; name="%s"' % key)
            _append_bytes(L, '')
            _append_bytes(L, value)
        else:
            _append_bytes(L, '--' + BOUNDARY)
            _append_bytes(L, 'Content-Disposition: form-data; name="%s"; filename="%s"' % (key, value.name))
            _append_bytes(L, 'Content-Type: %s' % value.content_type)
            _append_bytes(L, '')
            _append_bytes(L, value.content)
    _append_bytes(L, '--' + BOUNDARY + '--')
    _append_bytes(L, '')
    body = CRLF.join(L)
    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
    return content_type, body
451544
function template(string, data) {
  var proxyRegEx = /\{\{([^\}]+)?\}\}/g;
  return string.replace(proxyRegEx, function(_, key) {
    var keyParts = key.split('.');
    var value = data;
    for (var i = 0; i < keyParts.length; i++) {
      value = value[keyParts[i]];
    }
    return value || '';
  });
}