Input dictionary has to have the following keys:
        site_name
        it builds the correct dictionary for dao input and executes the dao
436526
def insertSite(self, businput):
        
        conn = self.dbi.connection()
        tran = conn.begin()
        try:
            siteobj = { # FIXME: unused?
                "site_name" : businput["site_name"]
            }
            businput["site_id"] = self.sm.increment(conn, "SEQ_SI", tran)
            self.sitein.execute(conn, businput, tran)
            tran.commit()
        except Exception as ex:
            if (str(ex).lower().find("unique constraint") != -1 or
                str(ex).lower().find("duplicate") != -1):
                # already exists, lets fetch the ID
                self.logger.warning("Ignoring unique constraint violation")
                self.logger.warning(ex)
            else:
                if tran:
                    tran.rollback()
                self.logger.exception(ex)
                raise
        finally:
            if tran:
                tran.close()
            if conn:
                conn.close()
768546
def input_(data, field_path):
    """"""
    data_obj = Data.objects.get(id=data['__id'])

    inputs = copy.deepcopy(data_obj.input)
    # XXX: Optimize by hydrating only the required field (major refactoring).
    hydrate_input_references(inputs, data_obj.process.input_schema)
    hydrate_input_uploads(inputs, data_obj.process.input_schema)

    return dict_dot(inputs, field_path)
340939
def set_dict_options(self, options):
        
        if isinstance(options, dict):
            for key, option_data in options.items():
                self.set_options(key, option_data)
        else:
            raise OptionTypeError("Not An Accepted Input Format: %s. Must be Dictionary" %type(options))
1072710
def build_inside(input_method, input_args=None, substitutions=None):
    
    def process_keyvals(keyvals):
        """ ["key=val", "x=y"] -> {"key": "val", "x": "y"} """
        keyvals = keyvals or []
        processed_keyvals = {}
        for arg in keyvals:
            key, value = arg.split("=", 1)
            processed_keyvals[key] = value
        return processed_keyvals

    main = __name__.split('.', 1)[0]
    log_encoding = get_logging_encoding(main)
    logger.info("log encoding: %s", log_encoding)

    if not input_method:
        raise RuntimeError("No input method specified!")

    logger.debug("getting build json from input %s", input_method)

    cleaned_input_args = process_keyvals(input_args)
    cleaned_input_args['substitutions'] = process_keyvals(substitutions)

    input_runner = InputPluginsRunner([{'name': input_method,
                                        'args': cleaned_input_args}])
    build_json = input_runner.run()[input_method]

    if isinstance(build_json, Exception):
        raise RuntimeError("Input plugin raised exception: {}".format(build_json))
    logger.debug("build json: %s", build_json)
    if not build_json:
        raise RuntimeError("No valid build json!")
    if not isinstance(build_json, dict):
        raise RuntimeError("Input plugin did not return valid build json: {}".format(build_json))

    dbw = DockerBuildWorkflow(**build_json)
    try:
        build_result = dbw.build_docker_image()
    except Exception as e:
        logger.error('image build failed: %s', e)
        raise
    else:
        if not build_result or build_result.is_failed():
            raise RuntimeError("no image built")
        else:
            logger.info("build has finished successfully \\o/")
1083661
def _ingest_dict(self, input_dict, schema_dict, path_to_root):

        '''
            
        '''

        valid_dict = {}

    # construct path to root for rules
        rules_path_to_root = re.sub('\[\d+\]', '[0]', path_to_root)

    # iterate over keys in schema dict
        for key, value in schema_dict.items():
            key_path = path_to_root
            if not key_path == '.':
                key_path += '.'
            key_path += key
            rules_key_path = re.sub('\[\d+\]', '[0]', key_path)
            value_match = False
            if key in input_dict.keys():
                value_index = self._datatype_classes.index(value.__class__)
                value_type = self._datatype_names[value_index]
                try:
                    v_index = self._datatype_classes.index(input_dict[key].__class__)
                    v_type = self._datatype_names[v_index]
                    if v_type == value_type:
                        value_match = True
                except:
                    value_match = False
            if value_match:
                if value_type == 'null':
                    valid_dict[key] = input_dict[key]
                elif value_type == 'boolean':
                    valid_dict[key] = self._ingest_boolean(input_dict[key], key_path)
                elif value_type == 'number':
                    valid_dict[key] = self._ingest_number(input_dict[key], key_path)
                elif value_type == 'string':
                    valid_dict[key] = self._ingest_string(input_dict[key], key_path)
                elif value_type == 'map':
                    valid_dict[key] = self._ingest_dict(input_dict[key], schema_dict[key], key_path)
                elif value_type == 'list':
                    valid_dict[key] = self._ingest_list(input_dict[key], schema_dict[key], key_path)
            else:
                value_type = self.keyMap[rules_key_path]['value_datatype']
                if 'default_value' in self.keyMap[rules_key_path]:
                    valid_dict[key] = self.keyMap[rules_key_path]['default_value']
                elif value_type == 'null':
                    valid_dict[key] = None
                elif value_type == 'boolean':
                    valid_dict[key] = False
                elif value_type == 'number':
                    valid_dict[key] = 0.0
                    if 'integer_data' in self.keyMap[rules_key_path].keys():
                        if self.keyMap[rules_key_path]['integer_data']:
                            valid_dict[key] = 0
                elif value_type == 'string':
                    valid_dict[key] = ''
                elif value_type == 'list':
                    valid_dict[key] = []
                elif value_type == 'map':
                    valid_dict[key] = self._ingest_dict({}, schema_dict[key], key_path)

    # add extra fields if set to True
        if self.keyMap[rules_path_to_root]['extra_fields']:
            for key, value in input_dict.items():
                if key not in valid_dict.keys():
                    valid_dict[key] = value

        return valid_dict
704130
def build_model_input(cls, name='input'):
        
        return cls(name, PortDirection.INPUT, type=PortType.MODEL)
791606
def create_from_dictionary(self, datas):
        
        configuration = ObjectConfiguration()

        if "uri" in datas:
            configuration.uri = str(datas["uri"])
        if "title" in datas:
            configuration.title = str(datas["title"])
        if "description" in datas:
            configuration.description = str(datas["description"])

        return configuration
998756
def _init_cat_dict(self, cat_dict_class, key_in_self, **kwargs):
        
        # Catch errors associated with crappy, but not unexpected data
        try:
            new_entry = cat_dict_class(self, key=key_in_self, **kwargs)
        except CatDictError as err:
            if err.warn:
                self._log.info("'{}' Not adding '{}': '{}'".format(self[
                    self._KEYS.NAME], key_in_self, str(err)))
            return None
        return new_entry
510430
def _input_to_dict(i, records, remapped=None):
    
    if not remapped: remapped = {}
    var_type, records = _to_variable_type(i["type"], records)
    if var_type.startswith("Array") and "inputBinding" in i.get("type", {}):
        ib = i["type"]["inputBinding"]
    elif "inputBinding" in i:
        ib = i["inputBinding"]
    else:
        ib = {"prefix": None, "itemSeparator": ";;", "position": None}
    name = _id_to_localname(i["id"]) if "id" in i else i["name"]
    return {"name": remapped.get(name) or name,
            "variable_type": var_type,
            "prefix": ib["prefix"], "separator": ib["itemSeparator"],
            "position": ib["position"], "is_required": True,
            "default": i.get("default", None), "separate": ib.get("separate", True)}, records
1202866
def _connect_params_dict(self):
        """"""
        conn_dict = {
            "hostname": self.host,
            "port": self.port,
            "username": self.username,
            "password": self.password,
            "look_for_keys": self.use_keys,
            "allow_agent": self.allow_agent,
            "key_filename": self.key_file,
            "pkey": self.pkey,
            "passphrase": self.passphrase,
            "timeout": self.timeout,
            "auth_timeout": self.auth_timeout,
        }

        # Check if using SSH 'config' file mainly for SSH proxy support
        if self.ssh_config_file:
            conn_dict = self._use_ssh_config(conn_dict)
        return conn_dict
1140933
def load_site(srcdir, datadir, site_name=None):
    
    if site_name is None:
        site_name = 'primary'
    if not validate.valid_name(site_name):
        raise DatacatsError('{} is not a valid site name.'.format(site_name))

    cp = ConfigParser.SafeConfigParser()
    try:
        cp.read([srcdir + '/.datacats-environment'])
    except ConfigParser.Error:
        raise DatacatsError('Error reading environment information')

    site_section = 'site_' + site_name
    try:
        port = cp.getint(site_section, 'port')
    except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
        port = None
    try:
        address = cp.get(site_section, 'address')
    except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
        address = None
    try:
        site_url = cp.get(site_section, 'site_url')
    except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
        site_url = None

    passwords = {}
    cp = ConfigParser.SafeConfigParser()
    cp.read(datadir + '/sites/' + site_name + '/passwords.ini')
    try:
        pw_options = cp.options('passwords')
    except ConfigParser.NoSectionError:
        pw_options = []

    for n in pw_options:
        passwords[n.upper()] = cp.get('passwords', n)

    return port, address, site_url, passwords
1185494
def build

      if !self.change_address
        raise MissingChangeAddressError
      end

      result = Result.new

      add_input_for_utxo = proc do |utxo|
        raise ArgumentError, "Unspent output must contain index" if !utxo.index
        raise ArgumentError, "Unspent output must contain transaction_hash" if !utxo.transaction_hash

        if !self.internal_provider.consumed_unspent_output?(utxo)
          self.internal_provider.consume_unspent_output(utxo)

          result.inputs_amount += utxo.value
          txin = TransactionInput.new(
                                      previous_hash: utxo.transaction_hash,
                                      previous_index: utxo.index,
                                      # put the output script here so the signer knows which key to use.
                                      signature_script: utxo.script)
          txin.transaction_output = utxo
          result.transaction.add_input(txin)
        else
          # UTXO was already consumed possibly by another Tx Builder sharing the same provider.
        end
      end

      # If no outputs are specified, spend all utxos to a change address, minus the mining fee.
      if (self.outputs || []).size == 0
        result.inputs_amount = 0

        (self.prepended_unspent_outputs || []).each(&add_input_for_utxo)
        (self.unspent_outputs || []).each(&add_input_for_utxo)

        if result.transaction.inputs.size == 0
          raise MissingUnspentOutputsError, "Missing unspent outputs"
        end

        # Value will be determined after computing the fee
        change_output = TransactionOutput.new(value: 0, script: self.change_address.public_address.script)
        result.transaction.add_output(change_output)

        result.fee = compute_fee_for_transaction(result.transaction, self.fee_rate)
        result.outputs_amount = result.inputs_amount - result.fee
        result.change_amount = 0

        # Check if inputs cover the fees
        if result.outputs_amount < 0
          raise InsufficientFundsError.new(result)
        end

        # Warn if the output amount is relatively small.
        if result.outputs_amount < result.fee
          Diagnostics.current.add_message("BTC::TransactionBuilder: Warning! Spending all unspent outputs returns less than a mining fee. (#{result.outputs_amount} < #{result.fee})")
        end

        # Set the output value as needed
        result.transaction.outputs[0].value = result.outputs_amount

        # For each address that is a WIF, locate the matching input and sign it.
        attempt_to_sign_transaction(result)

        return result
      end # if no specific outputs required

      # We are having one or more outputs (e.g. normal payment)
      # Need to find appropriate unspents and compose a transaction.

      # Prepare all outputs.
      # result.outputs_amount will also contain a fee after it's calculated.
      (self.outputs || []).each do |txout|
        result.outputs_amount += txout.value
        result.transaction.add_output(txout)
      end

      # We'll determine final change value depending on inputs.
      # Setting default to MAX_MONEY will protect against a bug when we fail to update the amount and
      # spend unexpected amount on mining fees.
      change_output = TransactionOutput.new(value: MAX_MONEY, script: self.change_address.public_address.script)
      result.transaction.add_output(change_output)

      mandatory_utxos = (self.prepended_unspent_outputs || []).to_a.dup

      # We have specific outputs with specific amounts, so we need to select the best amount of coins.
      # To play nice with BIP32/BIP44 change addresses (that need to monitor an increasing amount of addresses),
      # we'll spend oldest outputs first and will add more and more newer outputs until we cover fees
      # and change output is either zero or above the dust limit.
      # If `confirmations` attribute is nil, order is preserved.
      utxos = self.unspent_outputs || []
      if self.keep_unspent_outputs_order
        sorted_utxos = utxos.to_a.dup
      else
        sorted_utxos = utxos.to_a.sort_by{|txout| -(txout.confirmations || -1) } # oldest first
      end
      
      if self.parent_transactions
        # Can repeat some outputs in mandatory_utxos or sorted_utxos, 
        # but double-spending will be prevented by provider.
        self.parent_transactions.each do |parenttx|
          sorted_utxos += parenttx.outputs
        end
      end
      
      all_utxos = (sorted_utxos + mandatory_utxos)

      while true
        if (sorted_utxos.size + mandatory_utxos.size) == 0
          raise InsufficientFundsError.new(result)
        end

        utxo = nil
        while (sorted_utxos.size + mandatory_utxos.size) > 0
          utxo = if mandatory_utxos.size > 0
            mandatory_utxos.shift
          else
            sorted_utxos.shift
          end 
          if utxo.value > 0 &&
            !utxo.script.op_return_script? &&
            !self.internal_provider.consumed_unspent_output?(utxo)
            self.internal_provider.consume_unspent_output(utxo)
            break
          else
            #puts "CONSUMED UTXO, SKIPPING: #{utxo.transaction_id}:#{utxo.index} (#{utxo.value})"
            # Continue reading utxos to find one that is not consumed yet.
            utxo = nil
          end
        end
        
        if !utxo
          # puts "ALL UTXOS:"
          # all_utxos.each do |utxo|
          #   puts "--> #{utxo.transaction_id}:#{utxo.index} (#{utxo.value})"
          # end
          raise InsufficientFundsError
        end
        
        result.inputs_amount += utxo.value

        raise ArgumentError, "Transaction hash must be non-nil in unspent output" if !utxo.transaction_hash
        raise ArgumentError, "Index must be non-nil in unspent output" if !utxo.index

        txin = TransactionInput.new(previous_hash: utxo.transaction_hash,
                                    previous_index: utxo.index,
                                    # put the output script here so the signer knows which key to use.
                                    signature_script: utxo.script)
        txin.transaction_output = utxo
        result.transaction.add_input(txin)

        # Do not check the result before we included all the mandatory utxos.
        if mandatory_utxos.size == 0
          # Before computing the fee, quick check if we have enough inputs to cover the outputs.
          # If not, go and add one more utxo before wasting time computing fees.
          if result.inputs_amount >= result.outputs_amount
            fee = compute_fee_for_transaction(result.transaction, self.fee_rate)

            change = result.inputs_amount - result.outputs_amount - fee

            if change >= self.minimum_change

              # We have a big enough change, set missing values and return.

              change_output.value = change
              result.change_amount = change
              result.outputs_amount += change
              result.fee = fee
              attempt_to_sign_transaction(result)
              return result

            elsif change > self.dust_change && change < self.minimum_change

              # We have a shitty change: not small enough to forgo, not big enough to be useful.
              # Try adding more utxos on the next cycle (or fail if no more utxos are available).

            elsif change >= 0 && change <= self.dust_change
              # This also includes the case when change is exactly zero satoshis.
              # Remove the change output, keep existing outputs_amount, set fee and try to sign.
              result.transaction.outputs = result.transaction.outputs[0, result.transaction.outputs.size - 1]
              result.change_amount = 0
              result.fee = fee
              attempt_to_sign_transaction(result)
              return result

            else
              # Change is negative, we need more funds for this transaction.
              # Try adding more utxos on the next cycle.
              result.fee = fee
            end

          end # if inputs_amount >= outputs_amount
        end # if no more mandatory outputs to add
      end # while true
    end
341846
def dict_to_op(d, index_name, doc_type, op_type='index'):
    
    if d is None:
        return d

    op_types = ('create', 'delete', 'index', 'update')
    if op_type not in op_types:
        msg = 'Unknown operation type "{}", must be one of: {}'
        raise Exception(msg.format(op_type, ', '.join(op_types)))

    if 'id' not in d:
        raise Exception('"id" key not found')

    operation = {
        '_op_type': op_type,
        '_index': index_name,
        '_type': doc_type,
        '_id': d.pop('id'),
    }
    operation.update(d)

    return operation
332271
def _map_arguments(self, input_dict):
        
        data = input_dict.get('data')
        comp = input_dict.get('comp')
        library = input_dict.get('library')
        models = input_dict.get('models')
        hpx_order = input_dict.get('hpx_order_fitting')
        dry_run = input_dict.get('dry_run', False)

        self._set_link('init-model', InitModel,
                       comp=comp, data=data,
                       library=library,
                       models=models,
                       hpx_order=hpx_order,
                       dry_run=dry_run)

        self._set_link('assemble-model', AssembleModel_SG,
                       comp=comp, data=data,
                       models=models)
330711
def console_create():
    
    import argparse
    import getpass
    import re
    import sys
    import requests
    from six.moves import input
    from koordinates.client import Client

    parser = argparse.ArgumentParser(description="Command line tool to create a Koordinates API Token.")
    parser.add_argument('site', help="Domain (eg. labs.koordinates.com) for the Koordinates site.", metavar='SITE')
    parser.add_argument('email', help="User account email address", metavar='EMAIL')
    parser.add_argument('name', help="Description for the key", metavar='NAME')
    parser.add_argument('--scopes', help="Scopes for the new API token", nargs='+', metavar="SCOPE")
    parser.add_argument('--referrers', help="Restrict the request referrers for the token. You can use * as a wildcard, eg. *.example.com", nargs='+', metavar='HOST')
    parser.add_argument('--expires', help="Expiry time in ISO 8601 (YYYY-MM-DD) format", metavar="DATE")
    args = parser.parse_args()

    # check we have a valid-ish looking domain name
    if not re.match(r'(?=^.{4,253}$)(^((?!-)[a-zA-Z0-9-]{1,63}(?<!-)\.)+[a-zA-Z]{2,63}$)', args.site):
        parser.error("'%s' doesn't look like a valid domain name." % args.site)

    # check we have a valid-ish looking email address
    if not re.match(r'[^@]+@[^@]+\.[^@]+$', args.email):
        parser.error("'%s' doesn't look like a valid email address." % args.email)

    password = getpass.getpass('Account password for %s: ' % args.email)
    if not password:
        parser.error("Empty password.")

    expires_at = make_date(args.expires)

    print("\nNew API Token Parameters:")
    print("  Koordinates Site: %s" % args.site)
    print("  User email address: %s" % args.email)
    print("  Name: %s" % args.name)
    print("  Scopes: %s" % ' '.join(args.scopes or ['(default)']))
    print("  Referrer restrictions: %s" % ' '.join(args.referrers or ['(none)']))
    print("  Expires: %s" % (expires_at or '(never)'))
    if input("Continue? [Yn] ").lower() == 'n':
        sys.exit(1)

    token = Token(name=args.name)
    if args.scopes:
        token.scopes = args.scopes
    if args.referrers:
        token.referrers = args.referrers
    if expires_at:
        token.expires_at = expires_at

    print("\nRequesting token...")
    # need a dummy token here for initialisation
    client = Client(host=args.site, token='-dummy-')
    try:
        token = client.tokens.create(token, args.email, password)
    except requests.HTTPError as e:
        print("%s: %s" % (e, e.response.text))

        # Helpful tips for specific errors
        if e.response.status_code == 401:
            print("  => Check your email address, password, and site domain carefully.")

        sys.exit(1)

    print("Token created successfully.\n  Key: %s\n  Scopes: %s" % (token.key, token.scope))
791698
def _preprocess(project_dict):
        
        handlers = {
            ('archive',): _list_if_none,
            ('on-run-start',): _list_if_none_or_string,
            ('on-run-end',): _list_if_none_or_string,
        }

        for k in ('models', 'seeds'):
            handlers[(k,)] = _dict_if_none
            handlers[(k, 'vars')] = _dict_if_none
            handlers[(k, 'pre-hook')] = _list_if_none_or_string
            handlers[(k, 'post-hook')] = _list_if_none_or_string
        handlers[('seeds', 'column_types')] = _dict_if_none

        def converter(value, keypath):
            if keypath in handlers:
                handler = handlers[keypath]
                return handler(value)
            else:
                return value

        return deep_map(converter, project_dict)
730448
def execute(self, conn,  origin_site_name="", dataset="", block_name="", transaction = False):
        
        if not conn:
            dbsExceptionHandler("dbsException-db-conn-failed",
                                "Oracle/Block/List.  Expects db connection from upper layer.", self.logger.exception)
        binds = {}
        if origin_site_name:
            wheresql = 'WHERE B.ORIGIN_SITE_NAME = :origin_site_name'
            binds.update(origin_site_name=origin_site_name)

        if dataset:
            if 'wheresql' in locals():
                wheresql += ' AND DS.DATASET = :dataset'
            else:
                wheresql = 'WHERE DS.DATASET = :dataset'
            binds.update(dataset=dataset)

        if block_name:
            if 'wheresql' in locals():
                wheresql += ' AND B.BLOCK_NAME = :block_name'
            else:
                wheresql = 'WHERE B.BLOCK_NAME = :block_name'
            binds.update(block_name=block_name)

        sql = '{sql} {wheresql}'.format(sql=self.sql, wheresql=wheresql)

        cursors = self.dbi.processData(sql, binds, conn, transaction, returnCursor=True)
        result = []
        for cursor in cursors:
            result.extend(self.formatCursor(cursor, size=100))
        return result