Extract the header of the Clublog XML File
499983
def _extract_clublog_header(self, cty_xml_filename):
        

        cty_header = {}

        try:
            with open(cty_xml_filename, "r") as cty:
                raw_header = cty.readline()

            cty_date = re.search("date='.+'", raw_header)
            if cty_date:
                cty_date = cty_date.group(0).replace("date=", "").replace("'", "")
                cty_date = datetime.strptime(cty_date[:19], '%Y-%m-%dT%H:%M:%S')
                cty_date.replace(tzinfo=UTC)
                cty_header["Date"] = cty_date

            cty_ns = re.search("xmlns='.+[']", raw_header)
            if cty_ns:
                cty_ns = cty_ns.group(0).replace("xmlns=", "").replace("'", "")
                cty_header['NameSpace'] = cty_ns

            if len(cty_header) == 2:
                self._logger.debug("Header successfully retrieved from CTY File")
            elif len(cty_header) < 2:
                self._logger.warning("Header could only be partically retrieved from CTY File")
                self._logger.warning("Content of Header: ")
                for key in cty_header:
                    self._logger.warning(str(key)+": "+str(cty_header[key]))
            return cty_header

        except Exception as e:
            self._logger.error("Clublog CTY File could not be opened / modified")
            self._logger.error("Error Message: " + str(e))
            return
846191
private function extractHeaders($str)
    {
        $this->debug->log(__METHOD__);
        $this->debug->groupUncollapse();
        $headerStart = 0;
        #$this->debug->log(('top', \substr($return, 0, $this->curlInfo['header_size']));
        $headerLenStack = array();
        $i = 0;
        while (true) {
            $headerEnd     = \strpos($str, "\r\n\r\n", $headerStart);
            $headerLength  = $headerEnd - $headerStart;
            $headers       = \substr($str, $headerStart, $headerLength);
            $headerStart   = $headerEnd+4;
            $headerLenStack[] = \strlen($headers)+4;
            #$this->debug->log('headers', \str_replace(array("\r","\n"), array('|','|'), $headers));
            if (\count($headerLenStack) > $this->curlInfo['redirect_count']) {
                #$this->debug->log('header_len_stack','['.\implode(',',$header_len_stack).']');
                $sum = 0;
                for ($j=$i; $j>=0; $j--) {
                    $sum += $headerLenStack[$j];
                    if ($sum == $this->curlInfo['header_size']) {
                        #$this->debug->info('found headers');
                        break 2;
                    }
                }
            }
            if ($headerEnd === false || $i > 20) {
                $this->debug->warn('error parsing headers');
                break;
            }
            $i++;
        }
        // $this->debug->log('headers', $headers);
        return $headers;
    }
4582
def club(self,cid):
        '''
        
        '''
        headers = {"Content-type": "application/x-www-form-urlencoded","Accept": "text/plain",'Referer': 'http://'+self.domain+'/',"User-Agent": user_agent}
        req = self.session.get('http://'+self.domain+'/clubInfo.phtml?cid='+cid,headers=headers).content
        soup = BeautifulSoup(req)
        plist = []
        for i in soup.find('table',cellpadding=2).find_all('tr')[1:]:
            plist.append('%s\t%s\t%s\t%s\t%s'%(i.find_all('td')[0].text,i.find_all('td')[1].text,i.find_all('td')[2].text,i.find_all('td')[3].text,i.find_all('td')[4].text))
        return soup.title.text,plist
976986
@Override
    public Content getModuleHeader(String heading) {
        HtmlTree bodyTree = getBody(true, getWindowTitle(mdle.getQualifiedName().toString()));
        HtmlTree htmlTree = (configuration.allowTag(HtmlTag.HEADER))
                ? HtmlTree.HEADER()
                : bodyTree;
        addTop(htmlTree);
        addNavLinks(true, htmlTree);
        if (configuration.allowTag(HtmlTag.HEADER)) {
            bodyTree.addContent(htmlTree);
        }
        HtmlTree div = new HtmlTree(HtmlTag.DIV);
        div.addStyle(HtmlStyle.header);
        Content annotationContent = new HtmlTree(HtmlTag.P);
        addAnnotationInfo(mdle, annotationContent);
        div.addContent(annotationContent);
        Content label = mdle.isOpen() && (configuration.docEnv.getModuleMode() == ModuleMode.ALL)
                ? contents.openModuleLabel : contents.moduleLabel;
        Content tHeading = HtmlTree.HEADING(HtmlConstants.TITLE_HEADING, true,
                HtmlStyle.title, label);
        tHeading.addContent(Contents.SPACE);
        Content moduleHead = new RawHtml(heading);
        tHeading.addContent(moduleHead);
        div.addContent(tHeading);
        if (configuration.allowTag(HtmlTag.MAIN)) {
            mainTree.addContent(div);
        } else {
            bodyTree.addContent(div);
        }
        return bodyTree;
    }
189362
def get_headerReference(self, type_):
        """"""
        matching_headerReferences = self.xpath(
            "./w:headerReference[@w:type='%s']" % WD_HEADER_FOOTER.to_xml(type_)
        )
        if len(matching_headerReferences) == 0:
            return None
        return matching_headerReferences[0]
1271158
def _parse_clublog_xml(self, cty_xml_filename):
        

        entities = {}
        call_exceptions = {}
        prefixes = {}
        invalid_operations = {}
        zone_exceptions = {}

        call_exceptions_index = {}
        prefixes_index = {}
        invalid_operations_index = {}
        zone_exceptions_index = {}

        cty_tree = ET.parse(cty_xml_filename)
        root = cty_tree.getroot()

        #retrieve ADIF Country Entities
        cty_entities = cty_tree.find("entities")
        self._logger.debug("total entities: " + str(len(cty_entities)))
        if len(cty_entities) > 1:
            for cty_entity in cty_entities:
                try:
                    entity = {}
                    for item in cty_entity:
                        if item.tag == "name":
                            entity[const.COUNTRY] = unicode(item.text)
                            self._logger.debug(unicode(item.text))
                        elif item.tag == "prefix":
                            entity[const.PREFIX] = unicode(item.text)
                        elif item.tag == "deleted":
                            if item.text == "TRUE":
                                entity[const.DELETED] = True
                            else:
                                entity[const.DELETED] = False
                        elif item.tag == "cqz":
                            entity[const.CQZ] = int(item.text)
                        elif item.tag == "cont":
                            entity[const.CONTINENT] = unicode(item.text)
                        elif item.tag == "long":
                            entity[const.LONGITUDE] = float(item.text)
                        elif item.tag == "lat":
                            entity[const.LATITUDE] = float(item.text)
                        elif item.tag == "start":
                            dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                            entity[const.START] = dt.replace(tzinfo=UTC)
                        elif item.tag == "end":
                            dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                            entity[const.END] = dt.replace(tzinfo=UTC)
                        elif item.tag == "whitelist":
                            if item.text == "TRUE":
                                entity[const.WHITELIST] = True
                            else:
                                entity[const.WHITELIST] = False
                        elif item.tag == "whitelist_start":
                            dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                            entity[const.WHITELIST_START] = dt.replace(tzinfo=UTC)
                        elif item.tag == "whitelist_end":
                            dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                            entity[const.WHITELIST_END] = dt.replace(tzinfo=UTC)
                except AttributeError:
                    self._logger.error("Error while processing: ")
                entities[int(cty_entity[0].text)] = entity
            self._logger.debug(str(len(entities))+" Entities added")
        else:
            raise Exception("No Country Entities detected in XML File")


        cty_exceptions = cty_tree.find("exceptions")
        if len(cty_exceptions) > 1:
            for cty_exception in cty_exceptions:
                call_exception = {}
                for item in cty_exception:
                    if item.tag == "call":
                        call = str(item.text)
                        if call in call_exceptions_index.keys():
                            call_exceptions_index[call].append(int(cty_exception.attrib["record"]))
                        else:
                            call_exceptions_index[call] = [int(cty_exception.attrib["record"])]
                    elif item.tag == "entity":
                        call_exception[const.COUNTRY] = unicode(item.text)
                    elif item.tag == "adif":
                        call_exception[const.ADIF] = int(item.text)
                    elif item.tag == "cqz":
                        call_exception[const.CQZ] = int(item.text)
                    elif item.tag == "cont":
                        call_exception[const.CONTINENT] = unicode(item.text)
                    elif item.tag == "long":
                        call_exception[const.LONGITUDE] = float(item.text)
                    elif item.tag == "lat":
                        call_exception[const.LATITUDE] = float(item.text)
                    elif item.tag == "start":
                        dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                        call_exception[const.START] = dt.replace(tzinfo=UTC)
                    elif item.tag == "end":
                        dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                        call_exception[const.END] = dt.replace(tzinfo=UTC)
                    call_exceptions[int(cty_exception.attrib["record"])] = call_exception

            self._logger.debug(str(len(call_exceptions))+" Exceptions added")
            self._logger.debug(str(len(call_exceptions_index))+" unique Calls in Index ")

        else:
            raise Exception("No Exceptions detected in XML File")


        cty_prefixes = cty_tree.find("prefixes")
        if len(cty_prefixes) > 1:
            for cty_prefix in cty_prefixes:
                prefix = {}
                for item in cty_prefix:
                    pref = None
                    if item.tag == "call":

                        #create index for this prefix
                        call = str(item.text)
                        if call in prefixes_index.keys():
                            prefixes_index[call].append(int(cty_prefix.attrib["record"]))
                        else:
                            prefixes_index[call] = [int(cty_prefix.attrib["record"])]
                    if item.tag == "entity":
                        prefix[const.COUNTRY] = unicode(item.text)
                    elif item.tag == "adif":
                        prefix[const.ADIF] = int(item.text)
                    elif item.tag == "cqz":
                        prefix[const.CQZ] = int(item.text)
                    elif item.tag == "cont":
                        prefix[const.CONTINENT] = unicode(item.text)
                    elif item.tag == "long":
                        prefix[const.LONGITUDE] = float(item.text)
                    elif item.tag == "lat":
                        prefix[const.LATITUDE] = float(item.text)
                    elif item.tag == "start":
                        dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                        prefix[const.START] = dt.replace(tzinfo=UTC)
                    elif item.tag == "end":
                        dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                        prefix[const.END] = dt.replace(tzinfo=UTC)
                    prefixes[int(cty_prefix.attrib["record"])] = prefix

            self._logger.debug(str(len(prefixes))+" Prefixes added")
            self._logger.debug(str(len(prefixes_index))+" unique Prefixes in Index")
        else:
            raise Exception("No Prefixes detected in XML File")

        cty_inv_operations = cty_tree.find("invalid_operations")
        if len(cty_inv_operations) > 1:
            for cty_inv_operation in cty_inv_operations:
                invalid_operation = {}
                for item in cty_inv_operation:
                    call = None
                    if item.tag == "call":
                        call = str(item.text)
                        if call in invalid_operations_index.keys():
                            invalid_operations_index[call].append(int(cty_inv_operation.attrib["record"]))
                        else:
                            invalid_operations_index[call] = [int(cty_inv_operation.attrib["record"])]

                    elif item.tag == "start":
                        dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                        invalid_operation[const.START] = dt.replace(tzinfo=UTC)
                    elif item.tag == "end":
                        dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                        invalid_operation[const.END] = dt.replace(tzinfo=UTC)
                    invalid_operations[int(cty_inv_operation.attrib["record"])] = invalid_operation

            self._logger.debug(str(len(invalid_operations))+" Invalid Operations added")
            self._logger.debug(str(len(invalid_operations_index))+" unique Calls in Index")
        else:
            raise Exception("No records for invalid operations detected in XML File")


        cty_zone_exceptions = cty_tree.find("zone_exceptions")
        if len(cty_zone_exceptions) > 1:
            for cty_zone_exception in cty_zone_exceptions:
                zoneException = {}
                for item in cty_zone_exception:
                    call = None
                    if item.tag == "call":
                        call = str(item.text)
                        if call in zone_exceptions_index.keys():
                            zone_exceptions_index[call].append(int(cty_zone_exception.attrib["record"]))
                        else:
                            zone_exceptions_index[call] = [int(cty_zone_exception.attrib["record"])]

                    elif item.tag == "zone":
                        zoneException[const.CQZ] = int(item.text)
                    elif item.tag == "start":
                        dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                        zoneException[const.START] = dt.replace(tzinfo=UTC)
                    elif item.tag == "end":
                        dt = datetime.strptime(item.text[:19], '%Y-%m-%dT%H:%M:%S')
                        zoneException[const.END] = dt.replace(tzinfo=UTC)
                    zone_exceptions[int(cty_zone_exception.attrib["record"])] = zoneException

            self._logger.debug(str(len(zone_exceptions))+" Zone Exceptions added")
            self._logger.debug(str(len(zone_exceptions_index))+" unique Calls in Index")
        else:
            raise Exception("No records for zone exceptions detected in XML File")

        result = {
            "entities" : entities,
            "call_exceptions" : call_exceptions,
            "prefixes" : prefixes,
            "invalid_operations" : invalid_operations,
            "zone_exceptions" : zone_exceptions,
            "prefixes_index" : prefixes_index,
            "call_exceptions_index" : call_exceptions_index,
            "invalid_operations_index" : invalid_operations_index,
            "zone_exceptions_index" : zone_exceptions_index,
        }
        return result
889966
def extract_header_value(response, key)
        response.headers[key].class == Array ? response.headers[key].first.to_i : response.headers[key].to_i 
      end
476709
def xml(self, indent = ""):
        """"""
        xml = indent + "<OutputTemplate id=\"" + self.id + "\" format=\"" + self.formatclass.__name__ + "\"" + " label=\"" + self.label + "\""
        if self.formatclass.mimetype:
            xml +=" mimetype=\""+self.formatclass.mimetype+"\""
        if self.formatclass.schema:
            xml +=" schema=\""+clam.common.util.xmlescape(self.formatclass.schema)+"\""
        if self.filename:
            xml +=" filename=\""+clam.common.util.xmlescape(self.filename)+"\""
        if self.extension:
            xml +=" extension=\""+clam.common.util.xmlescape(self.extension)+"\""
        if self.parent:
            xml +=" parent=\""+clam.common.util.xmlescape(self.parent)+"\""
        if self.unique:
            xml +=" unique=\"yes\""
        else:
            xml +=" unique=\"no\""
        xml += ">\n"
        for metafield in self.metafields:
            xml += metafield.xml(indent + "    ") + "\n"

        xml += indent + "</OutputTemplate>"
        return xml
444180
def output_xml(data, code, headers=None):
    """"""
    resp = make_response(dumps({'response' :data}), code)
    resp.headers.extend(headers or {})
    return resp
1139193
function xmlString()
    {
        $doc = new DOMDocument( '1.0', 'utf-8' );
        $root = $doc->createElement( "ezmatrix" );
        $doc->appendChild( $root );

        $name = $doc->createElement( "name", $this->Name );
        $root->appendChild( $name );

        $columnsNode = $doc->createElement( "columns" );

        $sequentalColumns = $this->Matrix['columns']['sequential'];
        $columnAmount = $this->NumColumns;
        $columnsNode->setAttribute( 'number', $columnAmount );
        $root->appendChild( $columnsNode );

        if ( $sequentalColumns != null )
        {
            for( $i = 0; $i < $columnAmount; $i++ )
            {
                $column = $sequentalColumns[$i];
                if ( $column != null && $column['identifier'] != 'col_' . ($i+1) )
                {
                    unset( $columnNode );
                    $columnNode = $doc->createElement( 'column' );
                    $columnNode->appendChild( $doc->createTextNode( $column['name'] ) );
                    $columnNode->setAttribute( 'num', $i );
                    $columnNode->setAttribute( 'id', $column['identifier'] );

                    $columnsNode->appendChild( $columnNode );
                }
            }

        }

        $rowsNode =  $doc->createElement( "rows" );
        $rowAmount = $this->NumRows;

        $rowsNode->setAttribute( 'number', $rowAmount );

        $root->appendChild( $rowsNode );

        foreach ( $this->Cells as $cell )
        {
            unset( $cellNode );
            $cellNode = $doc->createElement( 'c' );
            $cellNode->appendChild( $doc->createTextNode( $cell ) );

            $root->appendChild( $cellNode );
        }

        return $this->domString( $doc );
    }
1303913
def as_log_output(self):
        
        elem = self.to_element()
        for attrname in self.sensitive_attributes:
            for sensitive_el in elem.iter(attrname):
                sensitive_el.text = 'XXXXXXXXXXXXXXXX'
        return ElementTreeBuilder.tostring(elem, encoding='UTF-8')
928588
public function extractHeaders()
    {
        $headers = [];
        foreach (array_keys($this->cors) as $headerField) {
            $headerData = $this->request->getHeaderLine($headerField);
            if ($headerData !== null) {
                $headers[$headerField] = $headerData;
            }
        }

        return $headers;
    }
188435
def do_header(self, node):
        """"""
        data = self.extract_text(node)
        self.add_text('\n/*\n %s \n*/\n' % data)
        # If our immediate sibling is a 'description' node then we
        # should comment that out also and remove it from the parent
        # node's children.
        parent = node.parentNode
        idx = parent.childNodes.index(node)
        if len(parent.childNodes) >= idx + 2:
            nd = parent.childNodes[idx + 2]
            if nd.nodeName == 'description':
                nd = parent.removeChild(nd)
                self.add_text('\n/*')
                self.subnode_parse(nd)
                self.add_text('\n*/\n')
1078506
def from_header_str(cls, header):
        
        if not header:
            return cls()

        try:
            params = header.strip().split(HEADER_DELIMITER)
            header_dict = {}
            data = {}

            for param in params:
                entry = param.split('=')
                key = entry[0]
                if key in (ROOT, PARENT, SAMPLE):
                    header_dict[key] = entry[1]
                # Ignore any "Self=" trace ids injected from ALB.
                elif key != SELF:
                    data[key] = entry[1]

            return cls(
                root=header_dict.get(ROOT, None),
                parent=header_dict.get(PARENT, None),
                sampled=header_dict.get(SAMPLE, None),
                data=data,
            )

        except Exception:
            log.warning("malformed tracing header %s, ignore.", header)
            return cls()
1078383
def parse_template(self, tmp, reset=False, only_body=False):
        
        logging.debug('GoT: """\n%s\n"""', tmp)

        if self.sent_time:
            self.modified_since_sent = True

        if only_body:
            self.body = tmp
        else:
            m = re.match(r'(?P<h>([a-zA-Z0-9_-]+:.+\n)*)\n?(?P<b>(\s*.*)*)',
                         tmp)
            assert m

            d = m.groupdict()
            headertext = d['h']
            self.body = d['b']

            # remove existing content
            if reset:
                self.headers = {}

            # go through multiline, utf-8 encoded headers
            # we decode the edited text ourselves here as
            # email.message_from_file can't deal with raw utf8 header values
            key = value = None
            for line in headertext.splitlines():
                if re.match('[a-zA-Z0-9_-]+:', line):  # new k/v pair
                    if key and value:  # save old one from stack
                        self.add(key, value)  # save
                    key, value = line.strip().split(':', 1)  # parse new pair
                    # strip spaces, otherwise we end up having " foo" as value
                    # of "Subject: foo"
                    value = value.strip()
                elif key and value:  # append new line without key prefix
                    value += line
            if key and value:  # save last one if present
                self.add(key, value)

            # interpret 'Attach' pseudo header
            if 'Attach' in self:
                to_attach = []
                for line in self.get_all('Attach'):
                    gpath = os.path.expanduser(line.strip())
                    to_attach += [g for g in glob.glob(gpath)
                                  if os.path.isfile(g)]
                logging.debug('Attaching: %s', to_attach)
                for path in to_attach:
                    self.attach(path)
                del self['Attach']
312349
private static String getHeaderInfo(String name, String headerLine) {
        String[] fields = headerLine.split("[\t ]");
        for (String field : fields) {
            if (field.startsWith(name + "=")) {
                String[] subfields = field.split("=");
                return subfields[1];
            }
        }
        return null;
    }
674284
function parseML(memberLog, required) {

  return memberLog.map(member => {

    const cloned = clone(member);
    const key = Reflect.ownKeys(cloned)[0];
    const value = cloned[key];

    cloned[key] = value.filter(elem => (elem !== required));

    return cloned;
  });
}