extract all keywords into two categories

            kws_ele: magnetic elements
            kws_bl: beamline elements

            return (kws_ele, kws_bl)
971304
def getAllKws(self):
        
        kws_ele = []
        kws_bl = []
        for ele in self.all_elements:
            if ele == '_prefixstr' or ele == '_epics':
                continue
            elif self.getElementType(ele).lower() == u'beamline':
                kws_bl.append(ele)
            else:
                kws_ele.append(ele)

        return tuple((kws_ele, kws_bl))
415430
def find_by_keyword(self, keywords="", strict_match=False, book=-1):
        self.fyi("find_by_keyword, ... book=%s" % book)
        ''''''
        self.fyi("nota.find_by_keyword() with keywords %s; book=%s" % (keywords, book))
        keywordsKnown = []
        if not strict_match:
            keywords[0] = keywords[0].lower()
        for k in self.cur.execute("SELECT keyword FROM keyword;").fetchall():
            if strict_match:
                keywordsKnown.extend(k)
            else:
                keywordsKnown.extend(k)
                keywordsKnown.extend([str(k[0]).lower()])
        self.fyi("keywordsKnown: %s" % keywordsKnown)
        # FIXME: only using first keyword here!
        if not strict_match:
            keywords_partial = []
            kl = len(keywords[0])
            if kl > 3:
                for K in keywordsKnown:
                    if K[0:kl] == keywords[0]:
                        if K not in keywords_partial:
                            keywords_partial.append(K)
            # Try fuzzy search only if no direct matches
            keywords_fuzzy = []
            if not len(keywords_partial):
                keywords_fuzzy = difflib.get_close_matches(keywords[0], keywordsKnown, n=1, cutoff=0.6)
            self.fyi("  keywords_partial %s" % keywords_partial)
            self.fyi("  keywords_fuzzy %s" % keywords_fuzzy)
            keywords = list(set(keywords_partial + keywords_fuzzy))
        self.fyi("nota.find_by_keyword() later, keywords: %s" % keywords)
        noteIds = []
        for keyword in keywords:
            self.fyi("keyword: %s" % keyword)
            if strict_match:
                self.fyi("strict match on keyword '%s'" % keyword)
                try:
                    keywordId = self.con.execute("SELECT keywordId FROM keyword WHERE keyword=?;",
                            [keyword]).fetchone()
                except:
                    self.error("cannot look up keyword '%s'" % [keyword])
            else:
                self.fyi("non-strict match on keyword '%s'" % keyword)
                try:
                    keywordId = []
                    for k in self.cur.execute("SELECT keywordId FROM keyword WHERE keyword=? COLLATE NOCASE;", [keyword]).fetchall():
                        keywordId.extend(k)
                except:
                    self.error("cannot look up keyword '%s'" % [keyword])
            try:
                if len(keywordId):
                    self.fyi("looking for noteID matches to keywordId %s" % keywordId)
                    for k in keywordId:
                        self.fyi("k: %s" % k)
                        try:
                            if strict_match:
                                noteIdtest = self.cur.execute("SELECT noteId FROM notekeyword WHERE keywordId=?;", [k])
                            else:
                                noteIdtest = self.cur.execute("SELECT noteId FROM notekeyword WHERE keywordId=? COLLATE NOCASE;", [k])
                        except:
                            self.error("cannot query database to find noteId corresponding to keywordId value %s" % [k])
                        for noteId in noteIdtest:
                            self.fyi("got match to noteID %s" % noteId)
                            if noteId not in noteIds:
                                noteIds.append(noteId)
                                self.fyi("adding to list")
                            else:
                                self.fyi("already in list")
                        self.fyi("done with keyword %s" % k)
                else:
                    pass
            except:
                #20150314 self.error("problem finding keyword or note in database")
                #20150314 shouldn't be an error to not find a note!
                pass
        ## convert from hash to ids. Note that one hash may create several ids.
        self.fyi("noteIds: %s" % noteIds)
        ## Find IDs of just the notes with the proper book value
        noteIds2 = []
        self.fyi("ORIGINAL noteIds: %s" % noteIds)
        for n in noteIds:
            self.fyi("n=%s" % n)
            try:
                row = self.cur.execute("SELECT noteId, book FROM note WHERE noteID=?;", n).fetchone()
            except:
                self.error("cannot look up noteId %s" % n)
            self.fyi("row %s; book=%s" % (row, book))
            if book < 0 or row[1] == book:
                self.fyi("appending id %s" % row[0])
                noteIds2.append((row[0],))
            else:
                self.fyi("skipping id %s because book is wrong" % row[0])
        noteIds = noteIds2
        #self.fyi("  LATER    noteIds2: %s" % noteIds2)
        #self.fyi("  LATER    noteIds: %s" % noteIds)
        rval = []
        for n in noteIds:
            #self.fyi(" processing id=%s" % n)
            try:
                note = self.cur.execute("SELECT noteId, authorId, date, title, content, due, privacy, modified, hash, book FROM note WHERE noteId=?;", n).fetchone()
            except:
                self.warning("Problem extracting note %s from database" % n)
                next
            if note:
                date = note[2]
                due = note[5]
                privacy = note[6]
                keywordIds = []
                keywordIds.extend(self.con.execute("SELECT keywordid FROM notekeyword WHERE notekeyword.noteid=?;", n))
                keywords = []
                for k in keywordIds:
                    keywords.append(self.cur.execute("SELECT keyword FROM keyword WHERE keywordId=?;", k).fetchone()[0])
                rval.append({"noteId":note[0], "title":note[3], "keywords":keywords,
                    "content":note[4], "due":note[5], "privacy":note[6],
                    "date":note[2], "modified":note[7], "hash":note[8], "book":note[9]})
        return rval
944357
def _parse_search_results_html(self, doc):
        
        subgroups = []
        soup = bs4.BeautifulSoup(doc, 'lxml')
        ele_divs = soup.select('div.item.prel')
        if not ele_divs:
            return subgroups
        for item in ele_divs:
            ele_a = item.select('p.tt > a')
            if not ele_a:
                continue
            link = ele_a[0].get('href')
            title = ele_a[0].get_text().strip()
            subgroups.append({
                'title': title,
                'link': link,
            })
        return subgroups
879817
def keywords2marc(self, key, values):
    
    result_695 = self.get('695', [])
    result_084 = self.get('084', [])
    result_6531 = self.get('6531', [])

    for value in values:
        schema = value.get('schema')
        source = value.get('source')
        keyword = value.get('value')

        if schema == 'PACS' or schema == 'PDG':
            result_084.append({
                '2': schema,
                '9': source,
                'a': keyword,
            })
        elif schema == 'JACOW':
            result_6531.append({
                '2': 'JACoW',
                '9': source,
                'a': keyword,
            })
        elif schema == 'INSPIRE':
            result_695.append({
                '2': 'INSPIRE',
                '9': source,
                'a': keyword,
            })
        elif schema == 'INIS':
            result_695.append({
                '2': 'INIS',
                '9': source,
                'a': keyword,
            })
        elif source != 'magpie':
            result_6531.append({
                '9': source,
                'a': keyword,
            })

    self['6531'] = result_6531
    self['084'] = result_084
    return result_695
1327972
def rinseElement(self, ele):
        
        if '*' in ele:
            tmplist = ''.join(ele.split()).split('*')
            tmplist_num = tmplist[[x.isdigit() for x in tmplist].index(True)]
            tmplist_ele = tmplist[[x.isdigit() for x in tmplist].index(False)]
            return dict(zip(('num', 'name'), (int(tmplist_num), tmplist_ele)))
        else:
            return dict(zip(('num', 'name'), (1, ele)))
1004569
function keywordsToArray(keywords, result) {
      if (typeof keywords === 'string') {
        result = result.concat(keywords.split(','));
      }
      else if (keywords instanceof Array) {
        if (keywords.length > 0) {
          for (var i = 0; i < keywords.length; i++) {
            result = keywordsToArray(keywords[i], result);
          }
        }
      }
      return result;
    }
487359
def get_keywords(self, entry):
        
        keyword_objects = []

        for keyword in entry.iterfind("./keyword"):
            identifier = keyword.get('id')
            name = keyword.text
            keyword_hash = hash(identifier)

            if keyword_hash not in self.keywords:
                self.keywords[keyword_hash] = models.Keyword(**{'identifier': identifier, 'name': name})

            keyword_objects.append(self.keywords[keyword_hash])

        return keyword_objects
1100151
function getKeywords(keywords) {
  let arrs = [];

  utils._each(keywords, (v, k) => {
    if (utils.isArray(v)) {
      let values = [];
      utils._each(v, (val) => {
        val = utils.getValueString('keywords.' + k, val);
        if (val) { values.push(val); }
      });
      v = values;
    } else {
      v = utils.getValueString('keywords.' + k, v);
      if (utils.isStr(v)) {
        v = [v];
      } else {
        return;
      } // unsuported types - don't send a key
    }
    arrs.push({key: k, value: v});
  });

  return arrs;
}
950393
def fetch_as_element(self, **kw):
        
        clone = self.copy()
        # Replace all filters with a combined filter
        bldata = TextFormat(field_format='name')
        # Preserve resolving fields for new filter
        if 'resolving' in self.format.data:
           bldata.set_resolving(**self.format.data['resolving'])

        # Resolve the entry ID to match SMC
        blid = TextFormat(field_format='pretty')
        blid.field_ids([LogField.BLACKLISTENTRYID])
        
        combined = CombinedFormat(bldata=bldata, blid=blid)
        clone.update_format(combined)

        for list_of_results in clone.fetch_raw(**kw):
            for entry in list_of_results:
                data = entry.get('bldata')
                data.update(**entry.get('blid'))
                yield BlacklistEntry(**data)
619732
def _extract_array(self, kwargs_list):
        
        lens_model = self._lens_mode_list[0]
        if self._solver_type == 'CENTER':
            center_x = kwargs_list[0]['center_x']
            center_y = kwargs_list[0]['center_y']
            x = [center_x, center_y]
        elif self._solver_type == 'ELLIPSE':
            e1 = kwargs_list[0]['e1']
            e2 = kwargs_list[0]['e2']
            x = [e1, e2]
        elif self._solver_type == 'SHAPELETS':
            coeffs = list(kwargs_list[0]['coeffs'])
            [c10, c01] = coeffs[1: 3]
            x = [c10, c01]
        elif self._solver_type == 'THETA_E_PHI':
            theta_E = kwargs_list[0]['theta_E']
            e1 = kwargs_list[1]['e1']
            e2 = kwargs_list[1]['e2']
            phi_ext, gamma_ext = param_util.ellipticity2phi_gamma(e1, e2)
            x = [theta_E, phi_ext]
        else:
            raise ValueError("Solver type %s not supported for 2-point solver!" % self._solver_type)
        return x
267246
def fromElement(cls, elem):
        
        self = cls()
        self.studyoid = elem.get("StudyOID")
        self.metadataversionoid = elem.get("MetaDataVersionOID")
        e_subjectdata = elem.findall(ODM_NS + "SubjectData")[0]
        self.subjectkey = e_subjectdata.get("SubjectKey")
        e_siteref = e_subjectdata.findall(ODM_NS + "SiteRef")[0]
        self.locationoid = e_siteref.get("LocationOID")

        e_links = e_subjectdata.findall(MEDI_NS + "Link")
        for e_link in e_links:
            self.links.append(e_link.get(XLINK_NS + "href"))

        decodes = {"yes": True, "no": False, "": None}
        for prop in RWSSubjectListItem.STATUS_PROPERTIES:
            val = e_subjectdata.get(MEDI_NS + prop, "").lower()
            setattr(self, prop.lower(), decodes.get(val, val))

        # By default we only get back active and non-deleted subjects
        self.active = decodes[
            e_subjectdata.get(MEDI_NS + "SubjectActive", "yes").lower()
        ]
        self.deleted = decodes[e_subjectdata.get(MEDI_NS + "Deleted", "no").lower()]

        return self
149455
def filter_core_keywords(keywords):
    """"""
    matches = {}
    for kw, info in keywords.items():
        if kw.core:
            matches[kw] = info
    return matches
849352
def _get_combined_keywords(_keywords, split_text):
    
    result = []
    _keywords = _keywords.copy()
    len_text = len(split_text)
    for i in range(len_text):
        word = _strip_word(split_text[i])
        if word in _keywords:
            combined_word = [word]
            if i + 1 == len_text:
                result.append(word)   # appends last word if keyword and doesn't iterate
            for j in range(i + 1, len_text):
                other_word = _strip_word(split_text[j])
                if other_word in _keywords and other_word == split_text[j] \
                        and other_word not in combined_word:
                    combined_word.append(other_word)
                else:
                    for keyword in combined_word:
                        _keywords.pop(keyword)
                    result.append(" ".join(combined_word))
                    break
    return result
221463
function keywords (schema, options) {
  options || (options = {})

  var target = options.target || 'keywords'    // Field to put the keywords array
    , source = options.source                  // Field to extract keywords from
    , minLength = options.minLength || 2       // Minimum keyword length
    , invalidChar = options.invalidChar || ''  // Invalid char replacement
    , naturalize = options.naturalize || false // Natural language stemming
    , fr = 'àáâãäåçèéêëìíîïñðóòôõöøùúûüýÿ'     // Accent chars to find
    , to = 'aaaaaaceeeeiiiinooooooouuuuyy'     // Accent replacement
    , fields = {}
    , PorterStemmer = require('./porter_stemmer')

  if (!(source instanceof Array)) source = [ source ]
  
  if (!schema.paths[target]) {
    fields[target] = [ String ]
  }

  for (var i = 0; i < source.length; i++) {
    if (!schema.paths[source[i]]) {
      fields[source[i]] = String
    }
  }

  schema.add(fields)
  
  schema.pre('save', function (next) {
    var words = []
    for (var i = 0; i < source.length; i++) {
      var add = this.extractKeywords(this[source[i]])
      for (var s = 0; s < add.length; s++) {
        if (!~words.indexOf(add[s])) words.push( add[s] )
      }
    }
    this[target] = words
    next()
  })
  
  ;['static'
  , 'method'
  ].forEach(function (method) {
    schema[method]('extractKeywords', function (str) {
      if (!str) return []
      str = str
        .replace(/^\s+|\s+$/g, '')
        .toLowerCase()
      
      for (var i=0; i<fr.length; i++) {
        str = str.replace(new RegExp(fr.charAt(i), 'g'), to.charAt(i))
      }
      str = str
        .replace(/[^a-z0-9 -]/g, invalidChar)
        .replace(new RegExp('['+invalidChar+']'+'+', 'g'), invalidChar)
        .split(/\s+/)
        .filter(function (v) { return v.length >= minLength })
        .filter(function (v, i, a) { return a.lastIndexOf(v) === i })
      
      if (naturalize) {
        str.forEach(function (v, i, arr) {
          arr[i] = PorterStemmer.stem(v)
        })
      }
      return str
    })
  })
}
1353015
def keywords(s, top=10, **kwargs):
    
    return parser.find_keywords(s, top=top, frequency=parser.frequency)
1101030
def _parse_keywords(self, prop):
        

        keywords = []

        if prop in KEYWORD_PROPS:
            xpath_root = self._data_map['_keywords_root']
            xpath_map = self._data_structures[prop]

            xtype = xpath_map['keyword_type']
            xpath = xpath_map['keyword']
            ktype = KEYWORD_TYPES[prop]

            for element in get_elements(self._xml_tree, xpath_root):
                if get_element_text(element, xtype).lower() == ktype.lower():
                    keywords.extend(get_elements_text(element, xpath))

        return keywords
4202
def translate_keyword_tokens( token )
      keywords = ["__LINE__", "__ENCODING__", "__FILE__", "BEGIN",   
                  "END", "alias", "and", "begin", "break", "case",
                  "class", "def", "defined?", "do", "else", "elsif",
                  "end", "ensure", "false", "for", "if", "in",
                  "module", "next", "nil", "not", "or", "redo",
                  "rescue", "retry", "return", "self", "super",
                  "then", "true", "undef", "unless", "until", 
                  "when", "while", "yield"]
      if keywords.include?( token.content )
        token.type = token.content.downcase.to_sym
        # Change the state if we match a keyword
        @expr_state = :beg
      end
      
      # A couple of exceptions      
      if token.content == "BEGIN"
        token.type = :begin_global 
        @expr_state = :beg
      elsif token.content == "END"
        token.type = :end_global 
        @expr_state = :beg
      end
      
      token
    end