static Document parseByteData(ByteBuffer byteData, String charsetName, String baseUri, Parser parser) {
    String docData;
    Document doc = null;

    // look for BOM - overrides any other header or input

    if (charsetName == null) { // determine from meta. safe parse as UTF-8
        // look for <meta http-equiv="Content-Type" content="text/html;charset=gb2312"> or HTML5 <meta charset="gb2312">
        docData = Charset.forName(defaultCharset).decode(byteData).toString();
        doc = parser.parseInput(docData, baseUri);
        Element meta = doc.select("meta[http-equiv=content-type], meta[charset]").first();
        if (meta != null) { // if not found, will keep utf-8 as best attempt
            String foundCharset = null;
            if (meta.hasAttr("http-equiv")) {
                foundCharset = getCharsetFromContentType(meta.attr("content"));
            }
            if (foundCharset == null && meta.hasAttr("charset")) {
                try {
                    if (Charset.isSupported(meta.attr("charset"))) {
                        foundCharset = meta.attr("charset");
                    }
                } catch (IllegalCharsetNameException e) {
                    foundCharset = null;
                }
            }

            if (foundCharset != null && foundCharset.length() != 0 && !foundCharset.equals(defaultCharset)) { // need to re-decode
                foundCharset = foundCharset.trim().replaceAll("[\"']", "");
                charsetName = foundCharset;
                byteData.rewind();
                docData = Charset.forName(foundCharset).decode(byteData).toString();
                doc = null;
            }
        }
    } else { // specified by content type header (or by user on file load)
        Validate.notEmpty(charsetName, "Must set charset arg to character set of file to parse. Set to null to attempt to detect from HTML");
        docData = Charset.forName(charsetName).decode(byteData).toString();
    }
    if (docData.length() > 0 && docData.charAt(0) == UNICODE_BOM) {
        byteData.rewind();
        docData = Charset.forName(defaultCharset).decode(byteData).toString();
        docData = docData.substring(1);
        charsetName = defaultCharset;
        doc = null;
    }
    if (doc == null) {
        doc = parser.parseInput(docData, baseUri);
        doc.outputSettings().charset(charsetName);
    }
    return doc;
}

static Document parseByteData(ByteBuffer byteData, String charsetName, String baseUri, Parser parser) {
    String docData;
    Document doc = null;

    // look for BOM - overrides any other header or input
    byteData.mark();
    byte[] bom = new byte[4];
    byteData.get(bom);
    byteData.rewind();
    if (bom[0] == 0x00 && bom[1] == 0x00 && bom[2] == (byte) 0xFE && bom[3] == (byte) 0xFF || // BE
            bom[0] == (byte) 0xFF && bom[1] == (byte) 0xFE && bom[2] == 0x00 && bom[3] == 0x00) { // LE
        charsetName = "UTF-32"; // and I hope it's on your system
    } else if (bom[0] == (byte) 0xFE && bom[1] == (byte) 0xFF || // BE
            bom[0] == (byte) 0xFF && bom[1] == (byte) 0xFE) {
        charsetName = "UTF-16"; // in all Javas
    } else if (bom[0] == (byte) 0xEF && bom[1] == (byte) 0xBB && bom[2] == (byte) 0xBF) {
        charsetName = "UTF-8"; // in all Javas
        byteData.position(3); // 16 and 32 decoders consume the BOM to determine be/le; utf-8 should be consumed
    }

    if (charsetName == null) { // determine from meta. safe parse as UTF-8
        // look for <meta http-equiv="Content-Type" content="text/html;charset=gb2312"> or HTML5 <meta charset="gb2312">
        docData = Charset.forName(defaultCharset).decode(byteData).toString();
        doc = parser.parseInput(docData, baseUri);
        Element meta = doc.select("meta[http-equiv=content-type], meta[charset]").first();
        if (meta != null) { // if not found, will keep utf-8 as best attempt
            String foundCharset = null;
            if (meta.hasAttr("http-equiv")) {
                foundCharset = getCharsetFromContentType(meta.attr("content"));
            }
            if (foundCharset == null && meta.hasAttr("charset")) {
                try {
                    if (Charset.isSupported(meta.attr("charset"))) {
                        foundCharset = meta.attr("charset");
                    }
                } catch (IllegalCharsetNameException e) {
                    foundCharset = null;
                }
            }

            if (foundCharset != null && foundCharset.length() != 0 && !foundCharset.equals(defaultCharset)) { // need to re-decode
                foundCharset = foundCharset.trim().replaceAll("[\"']", "");
                charsetName = foundCharset;
                byteData.rewind();
                docData = Charset.forName(foundCharset).decode(byteData).toString();
                doc = null;
            }
        }
    } else { // specified by content type header (or by user on file load)
        Validate.notEmpty(charsetName, "Must set charset arg to character set of file to parse. Set to null to attempt to detect from HTML");
        docData = Charset.forName(charsetName).decode(byteData).toString();
    }
    if (doc == null) {
        doc = parser.parseInput(docData, baseUri);
        doc.outputSettings().charset(charsetName);
    }
    return doc;
}

UTF16 streams with BOM are processed as UTF-8

The handling of the character encoding in org.jsoup.helper.DataUtil.parseByteData(...) is bugged when the input is an UTF16 stream with unicode BOM. This method does a check for presence of a BOM and, if it finds one, incorrectly assumes that this was a UTF-8 BOM. To fix this, the code would have to check the raw BOM bytes as the distinction between the various BOMs is lost after conversion to characters. See also: http://unicode.org/faq/utf_bom.html#bom4


boolean matchesLetter() {
    if (isEmpty())
        return false;
    char c = input[pos];
    return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
}

boolean matchesLetter() {
    if (isEmpty())
        return false;
    char c = input[pos];
    return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || Character.isLetter(c);
}

Problem in reading XML file containing Japanese tag names

Hello,
I have XML file containing Japanese tag names and values.
JSOUP is not parsing this Japanese tags.
I am using JSOUP library (version: 1.8.3).
Please help me to solve this issue.

e.g. ( XML File to reproduce problem )
<進捗推移グラフ>
<開始予定凡例名 表示状態="0" 線色="00CED1">①設計 開始予定</開始予定凡例名>
</進捗推移グラフ>
//// ****  Source Code  ******
Document doc = Jsoup.parse(XMLString.toString(),"UTF-8",Parser.xmlParser());
Elements objElementCollection = doc.getAllElements();
int iElementsSize=objElementCollection.size();
for(Element objCurrent : objElementCollection)
{
String szTag=objCurrent.tagName();
    for (TextNode tnTextNode : objCurrent.textNodes())
    {
        String szVal=tnTextNode.text();
    }

}


public String chompBalanced(char open, char close) {
    int start = -1;
    int end = -1;
    int depth = 0;
    char last = 0;

    do {
        if (isEmpty()) break;
        Character c = consume();
        if (last == 0 || last != ESC) {
            if (c.equals(open)) {
                depth++;
                if (start == -1)
                    start = pos;
            }
            else if (c.equals(close))
                depth--;
        }

        if (depth > 0 && last != 0)
            end = pos; // don't include the outer match pair in the return
        last = c;
    } while (depth > 0);
    return (end >= 0) ? queue.substring(start, end) : "";
}

public String chompBalanced(char open, char close) {
    int start = -1;
    int end = -1;
    int depth = 0;
    char last = 0;
    boolean inQuote = false;

    do {
        if (isEmpty()) break;
        Character c = consume();
        if (last == 0 || last != ESC) {
            if (c.equals('\'') || c.equals('"') && c != open)
                inQuote = !inQuote;
            if (inQuote)
                continue;
            if (c.equals(open)) {
                depth++;
                if (start == -1)
                    start = pos;
            }
            else if (c.equals(close))
                depth--;
        }

        if (depth > 0 && last != 0)
            end = pos; // don't include the outer match pair in the return
        last = c;
    } while (depth > 0);
    return (end >= 0) ? queue.substring(start, end) : "";
}

Parse failed with org.jsoup.select.Selector$SelectorParseException when selector has unbalanced '(' or '[' or ')' or ']'

Selector I am having as following div.card-content2:has(a.subtitle[title= MySubTitle:)]) OR a.title[title=MyTitle :] ]


private void copyAttributes(org.jsoup.nodes.Node source, Element el) {
    for (Attribute attribute : source.attributes()) {
        // valid xml attribute names are: ^[a-zA-Z_:][-a-zA-Z0-9_:.]
        String key = attribute.getKey().replaceAll("[^-a-zA-Z0-9_:.]", "");
            el.setAttribute(key, attribute.getValue());
    }
}

private void copyAttributes(org.jsoup.nodes.Node source, Element el) {
    for (Attribute attribute : source.attributes()) {
        // valid xml attribute names are: ^[a-zA-Z_:][-a-zA-Z0-9_:.]
        String key = attribute.getKey().replaceAll("[^-a-zA-Z0-9_:.]", "");
        if (key.matches("[a-zA-Z_:]{1}[-a-zA-Z0-9_:.]*"))
            el.setAttribute(key, attribute.getValue());
    }
}

INVALID_CHARACTER_ERR when converting Document to W3C

A recent ClearQuest version has an HTML generation bug, which is ignored by both Chrome and Internet Explorer. Jsoup.parse is also successful:
org.jsoup.nodes.Document doc = Jsoup.parse("");
(Please note the single quotation mark at the end of the body start tag.)
But trying to convert this to a W3C document fails:
new W3CDom().fromJsoup(doc);
Exception in thread "main" org.w3c.dom.DOMException: INVALID_CHARACTER_ERR: An invalid or illegal XML character is specified.
    at org.apache.xerces.dom.CoreDocumentImpl.createAttribute(Unknown Source)
    at org.apache.xerces.dom.ElementImpl.setAttribute(Unknown Source)
    at org.jsoup.helper.W3CDom$W3CBuilder.copyAttributes(W3CDom.java:124)
    at org.jsoup.helper.W3CDom$W3CBuilder.head(W3CDom.java:92)
    at org.jsoup.select.NodeTraversor.traverse(NodeTraversor.java:31)
    at org.jsoup.helper.W3CDom.convert(W3CDom.java:66)
    at org.jsoup.helper.W3CDom.fromJsoup(W3CDom.java:46)

Perhaps copyAttributes() should ignore invalid attributes, or catch exactly this error, and ignore it, or W3CDom could have flags to ignore such errors...


void read(Tokeniser t, CharacterReader r) {
    char c = r.consume();
    switch (c) {
        case '>':
            t.tagPending.selfClosing = true;
            t.emitTagPending();
            t.transition(Data);
            break;
        case eof:
            t.eofError(this);
            t.transition(Data);
            break;
        default:
            t.error(this);
            t.transition(BeforeAttributeName);
    }
}

void read(Tokeniser t, CharacterReader r) {
    char c = r.consume();
    switch (c) {
        case '>':
            t.tagPending.selfClosing = true;
            t.emitTagPending();
            t.transition(Data);
            break;
        case eof:
            t.eofError(this);
            t.transition(Data);
            break;
        default:
            t.error(this);
            r.unconsume();
            t.transition(BeforeAttributeName);
    }
}

Hello,
I don't know if it is a bug or not, but when I'm parsing:
<img /onerror="a()"/>
The result of the parsers is:
<img nerror="a()"/>
Is it OK? can I change the parser behavior for those types of tags?


public void removeIgnoreCase(String key) {
    Validate.notEmpty(key);
    if (attributes == null)
        return;
    for (Iterator<String> it = attributes.keySet().iterator(); it.hasNext(); ) {
        String attrKey = it.next();
        if (attrKey.equalsIgnoreCase(key))
            attributes.remove(attrKey);
    }
}

public void removeIgnoreCase(String key) {
    Validate.notEmpty(key);
    if (attributes == null)
        return;
    for (Iterator<String> it = attributes.keySet().iterator(); it.hasNext(); ) {
        String attrKey = it.next();
        if (attrKey.equalsIgnoreCase(key))
            it.remove();
    }
}

When testing out the removeIgnoreCase method, I'm now seeing a ConcurrentModificationException with code like: element.select("abc").first().removeAttr("attr1").removeAttr("attr2");
It appears to be due to using a foreach loop over the LinkedHashMap to do the removal. Changing to do the removal directly with an iterator fixes this issue.
Like so:
for (Iterator<Map.Entry<String, Attribute>> iter = attributes.entrySet().iterator(); iter.hasNext();) {
            Map.Entry<String, Attribute> entry = iter.next();
            if (entry.getKey().equalsIgnoreCase("key1")) {
                iter.remove();
            }
        }


final void newAttribute() {
    if (attributes == null)
        attributes = new Attributes();

    if (pendingAttributeName != null) {
        // the tokeniser has skipped whitespace control chars, but trimming could collapse to empty for other control codes, so verify here
        pendingAttributeName = pendingAttributeName.trim();
            Attribute attribute;
            if (hasPendingAttributeValue)
                attribute = new Attribute(pendingAttributeName,
                    pendingAttributeValue.length() > 0 ? pendingAttributeValue.toString() : pendingAttributeValueS);
            else if (hasEmptyAttributeValue)
                attribute = new Attribute(pendingAttributeName, "");
            else
                attribute = new BooleanAttribute(pendingAttributeName);
            attributes.put(attribute);
    }
    pendingAttributeName = null;
    hasEmptyAttributeValue = false;
    hasPendingAttributeValue = false;
    reset(pendingAttributeValue);
    pendingAttributeValueS = null;
}

final void newAttribute() {
    if (attributes == null)
        attributes = new Attributes();

    if (pendingAttributeName != null) {
        // the tokeniser has skipped whitespace control chars, but trimming could collapse to empty for other control codes, so verify here
        pendingAttributeName = pendingAttributeName.trim();
        if (pendingAttributeName.length() > 0) {
            Attribute attribute;
            if (hasPendingAttributeValue)
                attribute = new Attribute(pendingAttributeName,
                    pendingAttributeValue.length() > 0 ? pendingAttributeValue.toString() : pendingAttributeValueS);
            else if (hasEmptyAttributeValue)
                attribute = new Attribute(pendingAttributeName, "");
            else
                attribute = new BooleanAttribute(pendingAttributeName);
            attributes.put(attribute);
        }
    }
    pendingAttributeName = null;
    hasEmptyAttributeValue = false;
    hasPendingAttributeValue = false;
    reset(pendingAttributeValue);
    pendingAttributeValueS = null;
}

Jsoup.clean control characters throws: IllegalArgumentException: String must not be empty

I found that when running Jsoup.clean() on a string that contains the format below, Jsoup throws: IllegalArgumentException: String must not be empty.
The problematic string format:
'<a/*>', (where * is a control char).
i.e. < char followed by a letter (a-z), then any chars, / and any control char (ASCII 0-31) except 0, 9-10, 12-13, any chars, and a > char.


static String unescape(String string) {
    if (!string.contains("&"))
        return string;

    Matcher m = unescapePattern.matcher(string); // &(#(x|X)?([0-9a-fA-F]+)|[a-zA-Z]+);?
    StringBuffer accum = new StringBuffer(string.length()); // pity matcher can't use stringbuilder, avoid syncs
    // todo: replace m.appendReplacement with own impl, so StringBuilder and quoteReplacement not required

    while (m.find()) {
        int charval = -1;
        String num = m.group(3);
        if (num != null) {
            try {
                int base = m.group(2) != null ? 16 : 10; // 2 is hex indicator
                charval = Integer.valueOf(num, base);
            } catch (NumberFormatException e) {
            } // skip
        } else {
            String name = m.group(1);
            if (full.containsKey(name))
                charval = full.get(name);
        }

        if (charval != -1 || charval > 0xFFFF) { // out of range
            String c = Character.toString((char) charval);
            m.appendReplacement(accum, c);
        } else {
            m.appendReplacement(accum, m.group(0));
        }
    }
    m.appendTail(accum);
    return accum.toString();
}

static String unescape(String string) {
    if (!string.contains("&"))
        return string;

    Matcher m = unescapePattern.matcher(string); // &(#(x|X)?([0-9a-fA-F]+)|[a-zA-Z]+);?
    StringBuffer accum = new StringBuffer(string.length()); // pity matcher can't use stringbuilder, avoid syncs
    // todo: replace m.appendReplacement with own impl, so StringBuilder and quoteReplacement not required

    while (m.find()) {
        int charval = -1;
        String num = m.group(3);
        if (num != null) {
            try {
                int base = m.group(2) != null ? 16 : 10; // 2 is hex indicator
                charval = Integer.valueOf(num, base);
            } catch (NumberFormatException e) {
            } // skip
        } else {
            String name = m.group(1);
            if (full.containsKey(name))
                charval = full.get(name);
        }

        if (charval != -1 || charval > 0xFFFF) { // out of range
            String c = Character.toString((char) charval);
            m.appendReplacement(accum, Matcher.quoteReplacement(c));
        } else {
            m.appendReplacement(accum, Matcher.quoteReplacement(m.group(0))); // replace with original string
        }
    }
    m.appendTail(accum);
    return accum.toString();
}

StringIndexOutOfBoundsException when parsing link http://news.yahoo.com/s/nm/20100831/bs_nm/us_gm_china

java.lang.StringIndexOutOfBoundsException: String index out of range: 1
at java.lang.String.charAt(String.java:686)
at java.util.regex.Matcher.appendReplacement(Matcher.java:711)
at org.jsoup.nodes.Entities.unescape(Entities.java:69)
at org.jsoup.nodes.TextNode.createFromEncoded(TextNode.java:95)
at org.jsoup.parser.Parser.parseTextNode(Parser.java:222)
at org.jsoup.parser.Parser.parse(Parser.java:94)
at org.jsoup.parser.Parser.parse(Parser.java:54)
at org.jsoup.Jsoup.parse(Jsoup.java:30)


boolean anyOtherEndTag(Token t, HtmlTreeBuilder tb) {
    String name = t.asEndTag().normalName();
    ArrayList<Element> stack = tb.getStack();
    for (int pos = stack.size() -1; pos >= 0; pos--) {
        Element node = stack.get(pos);
        if (node.nodeName().equals(name)) {
            tb.generateImpliedEndTags(name);
            if (!name.equals(tb.currentElement().nodeName()))
                tb.error(this);
            tb.popStackToClose(name);
            break;
        } else {
            if (tb.isSpecial(node)) {
                tb.error(this);
                return false;
            }
        }
    }
    return true;
}

boolean anyOtherEndTag(Token t, HtmlTreeBuilder tb) {
    String name = t.asEndTag().name(); // matches with case sensitivity if enabled
    ArrayList<Element> stack = tb.getStack();
    for (int pos = stack.size() -1; pos >= 0; pos--) {
        Element node = stack.get(pos);
        if (node.nodeName().equals(name)) {
            tb.generateImpliedEndTags(name);
            if (!name.equals(tb.currentElement().nodeName()))
                tb.error(this);
            tb.popStackToClose(name);
            break;
        } else {
            if (tb.isSpecial(node)) {
                tb.error(this);
                return false;
            }
        }
    }
    return true;
}

Executing :
        String xml="<r><X>A</X><y>B</y></r>";
	Parser parser = Parser.htmlParser();
	parser.settings(ParseSettings.preserveCase);
	org.jsoup.nodes.Document _doc = parser.parseInput(xml, "/");

Results in :
<html>
<head></head>
<body>
<r>
<X>
A
<y>
B
</y>
</X>
</r>
</body>
</html>
Manual hacking : remove all .toLowerCase() invocations from Token.java (normalName=...)


private static void handleRawtext(Token.StartTag startTag, HtmlTreeBuilder tb) {
    tb.insert(startTag);
    tb.tokeniser.transition(TokeniserState.Rawtext);
    tb.markInsertionMode();
    tb.transition(Text);
}

private static void handleRawtext(Token.StartTag startTag, HtmlTreeBuilder tb) {
    tb.tokeniser.transition(TokeniserState.Rawtext);
    tb.markInsertionMode();
    tb.transition(Text);
    tb.insert(startTag);
}

Incorrect handling of self-closing tags noframes, style and title cause remainder of document to be html-escaped

Given the input:

       </head> <body> <p>Whatever</p> </body> </html>

With <title/> instead of


private boolean inSpecificScope(String[] targetNames, String[] baseTypes, String[] extraTypes) {
    // https://html.spec.whatwg.org/multipage/parsing.html#has-an-element-in-the-specific-scope
    int bottom = stack.size() -1;
    if (bottom > MaxScopeSearchDepth) {
        bottom = MaxScopeSearchDepth;
    }
    final int top = bottom > MaxScopeSearchDepth ? bottom - MaxScopeSearchDepth : 0;
    // don't walk too far up the tree

    for (int pos = bottom; pos >= top; pos--) {
        final String elName = stack.get(pos).nodeName();
        if (inSorted(elName, targetNames))
            return true;
        if (inSorted(elName, baseTypes))
            return false;
        if (extraTypes != null && inSorted(elName, extraTypes))
            return false;
    }
    //Validate.fail("Should not be reachable"); // would end up false because hitting 'html' at root (basetypes)
    return false;
}

private boolean inSpecificScope(String[] targetNames, String[] baseTypes, String[] extraTypes) {
    // https://html.spec.whatwg.org/multipage/parsing.html#has-an-element-in-the-specific-scope
    final int bottom = stack.size() -1;
    final int top = bottom > MaxScopeSearchDepth ? bottom - MaxScopeSearchDepth : 0;
    // don't walk too far up the tree

    for (int pos = bottom; pos >= top; pos--) {
        final String elName = stack.get(pos).nodeName();
        if (inSorted(elName, targetNames))
            return true;
        if (inSorted(elName, baseTypes))
            return false;
        if (extraTypes != null && inSorted(elName, extraTypes))
            return false;
    }
    //Validate.fail("Should not be reachable"); // would end up false because hitting 'html' at root (basetypes)
    return false;
}

version 1.10.3 no problem
version 1.11.1 java.lang.StackOverflowError
Example URL：
http://szshb.nxszs.gov.cn/
http://www.lnfsfda.gov.cn/
http://www.beihai.gov.cn/
http://www.fsepb.gov.cn/
http://www.bhem.gov.cn


static boolean preserveWhitespace(Node node) {
    // looks only at this element and five levels up, to prevent recursion & needless stack searches
    if (node != null && node instanceof Element) {
        Element el = (Element) node;
            if (el.tag.preserveWhitespace())
                return true;
            else
                return el.parent() != null && el.parent().tag.preserveWhitespace();
    }
    return false;
}

static boolean preserveWhitespace(Node node) {
    // looks only at this element and five levels up, to prevent recursion & needless stack searches
    if (node != null && node instanceof Element) {
        Element el = (Element) node;
        int i = 0;
        do {
            if (el.tag.preserveWhitespace())
                return true;
            el = el.parent();
            i++;
        } while (i < 6 && el != null);
    }
    return false;
}

Whitespaces not properly handled in <pre> tag

If a "pre" tag contains deep nested tags, whitespaces in nested tags are not preserved.
Example:
String s = "
\n"
        + "  message  other   \n    message  with \n"
        + "   whitespaces      \n"
        + "
";
    Document doc = Jsoup.parse(s);
    System.out.println(doc.select("pre").first().outerHtml());

Will output:

  message  other message with whiptespaces

Output is OK if we omit the "code" tag


private static String cacheString(final char[] charBuf, final String[] stringCache, final int start, final int count) {
    // limit (no cache):
    if (count > maxStringCacheLen)
        return new String(charBuf, start, count);

    // calculate hash:
    int hash = 0;
    int offset = start;
    for (int i = 0; i < count; i++) {
        hash = 31 * hash + charBuf[offset++];
    }

    // get from cache
    final int index = hash & stringCache.length - 1;
    String cached = stringCache[index];

    if (cached == null) { // miss, add
        cached = new String(charBuf, start, count);
        stringCache[index] = cached;
    } else { // hashcode hit, check equality
        if (rangeEquals(charBuf, start, count, cached)) { // hit
            return cached;
        } else { // hashcode conflict
            cached = new String(charBuf, start, count);
            stringCache[index] = cached; // update the cache, as recently used strings are more likely to show up again
        }
    }
    return cached;
}

private static String cacheString(final char[] charBuf, final String[] stringCache, final int start, final int count) {
    // limit (no cache):
    if (count > maxStringCacheLen)
        return new String(charBuf, start, count);
    if (count < 1)
        return "";

    // calculate hash:
    int hash = 0;
    int offset = start;
    for (int i = 0; i < count; i++) {
        hash = 31 * hash + charBuf[offset++];
    }

    // get from cache
    final int index = hash & stringCache.length - 1;
    String cached = stringCache[index];

    if (cached == null) { // miss, add
        cached = new String(charBuf, start, count);
        stringCache[index] = cached;
    } else { // hashcode hit, check equality
        if (rangeEquals(charBuf, start, count, cached)) { // hit
            return cached;
        } else { // hashcode conflict
            cached = new String(charBuf, start, count);
            stringCache[index] = cached; // update the cache, as recently used strings are more likely to show up again
        }
    }
    return cached;
}

StringIndexOutOfBoundsException as of jsoup 1.11.1


final void html(final Appendable accum, final Document.OutputSettings out) throws IOException {
    final int sz = size;
    for (int i = 0; i < sz; i++) {
        // inlined from Attribute.html()
        final String key = keys[i];
        final String val = vals[i];
        accum.append(' ').append(key);

        // collapse checked=null, checked="", checked=checked; write out others
        if (!(out.syntax() == Document.OutputSettings.Syntax.html
            && (val == null || val.equals(key) && Attribute.isBooleanAttribute(key)))) {
            accum.append("=\"");
            Entities.escape(accum, val == null ? EmptyString : val, out, true, false, false);
            accum.append('"');
        }
    }
}

final void html(final Appendable accum, final Document.OutputSettings out) throws IOException {
    final int sz = size;
    for (int i = 0; i < sz; i++) {
        // inlined from Attribute.html()
        final String key = keys[i];
        final String val = vals[i];
        accum.append(' ').append(key);

        // collapse checked=null, checked="", checked=checked; write out others
        if (!Attribute.shouldCollapseAttribute(key, val, out)) {
            accum.append("=\"");
            Entities.escape(accum, val == null ? EmptyString : val, out, true, false, false);
            accum.append('"');
        }
    }
}

Regression - Boolean attributes not collapsed when using HTML syntax

Hello,
First off, thanks for a really useful library.
So, upgrading from 1.10.2 to 1.11.2 we see that boolean attributes are no longer collapsed when using html syntax. Example test case:
    @Test
    public void test() {
        Document document = Jsoup.parse(
                "
");
        assertEquals("\n" +
                     " \n" +
                     " \n" +
                     "
\n" +
                     " \n" +
                     "",
                     document.outerHtml());
    }

Tracked it down to commit "Refactored Attributes to be an array pair vs LinkedHashSet " ea1fb65. The Attibutes.html(final Appendable accum, final Document.OutputSettings out) method no longer uses Attribute and fails to check the value of the attribute for an empty string(line 320).
If I may also suggest to use Attribute.shouldCollapseAttribute(String key, String val, Document.OutputSettings out) instead as a single source of truth as the boolean expression is complex enough and easy to make a mistake. Not sure if this would have an impact in performance though but I am guessing that optimizer will inline the call at some point anyways?


private void popStackToClose(Token.EndTag endTag) {
    String elName = endTag.name();
    Element firstFound = null;

    for (int pos = stack.size() -1; pos >= 0; pos--) {
        Element next = stack.get(pos);
        if (next.nodeName().equals(elName)) {
            firstFound = next;
            break;
        }
    }
    if (firstFound == null)
        return; // not found, skip

    for (int pos = stack.size() -1; pos >= 0; pos--) {
        Element next = stack.get(pos);
        stack.remove(pos);
        if (next == firstFound)
            break;
    }
}

private void popStackToClose(Token.EndTag endTag) {
    String elName = endTag.normalName();
    Element firstFound = null;

    for (int pos = stack.size() -1; pos >= 0; pos--) {
        Element next = stack.get(pos);
        if (next.nodeName().equals(elName)) {
            firstFound = next;
            break;
        }
    }
    if (firstFound == null)
        return; // not found, skip

    for (int pos = stack.size() -1; pos >= 0; pos--) {
        Element next = stack.get(pos);
        stack.remove(pos);
        if (next == firstFound)
            break;
    }
}

xmlParser() with ParseSettings.htmlDefault does not put end tag to lower case

@Test public void test() {
    Parser parser = Parser.xmlParser().settings(ParseSettings.htmlDefault);
    Document document = Jsoup.parse("
test

", "", parser);
    assertEquals("
\n test\n
\n

", document.toString()); // fail -> toString() = "
\n test\n

\n
"
}

@Test public void test1() {
    Parser parser = Parser.xmlParser().settings(ParseSettings.htmlDefault);
    Document document = Jsoup.parse("
test

", "", parser);
    assertEquals("
\n test\n
\n

", document.toString()); // pass
}


void insert(Token.Comment commentToken) {
    Comment comment = new Comment(commentToken.getData());
    Node insert = comment;
    if (commentToken.bogus) { // xml declarations are emitted as bogus comments (which is right for html, but not xml)
        // so we do a bit of a hack and parse the data as an element to pull the attributes out
        String data = comment.getData();
        if (data.length() > 1 && (data.startsWith("!") || data.startsWith("?"))) {
            Document doc = Jsoup.parse("<" + data.substring(1, data.length() -1) + ">", baseUri, Parser.xmlParser());
                Element el = doc.child(0);
                insert = new XmlDeclaration(settings.normalizeTag(el.tagName()), data.startsWith("!"));
                insert.attributes().addAll(el.attributes());
        }
    }
    insertNode(insert);
}

void insert(Token.Comment commentToken) {
    Comment comment = new Comment(commentToken.getData());
    Node insert = comment;
    if (commentToken.bogus) { // xml declarations are emitted as bogus comments (which is right for html, but not xml)
        // so we do a bit of a hack and parse the data as an element to pull the attributes out
        String data = comment.getData();
        if (data.length() > 1 && (data.startsWith("!") || data.startsWith("?"))) {
            Document doc = Jsoup.parse("<" + data.substring(1, data.length() -1) + ">", baseUri, Parser.xmlParser());
            if (doc.childNodeSize() > 0) {
                Element el = doc.child(0);
                insert = new XmlDeclaration(settings.normalizeTag(el.tagName()), data.startsWith("!"));
                insert.attributes().addAll(el.attributes());
            } // else, we couldn't parse it as a decl, so leave as a comment
        }
    }
    insertNode(insert);
}

Faulty Xml Causes IndexOutOfBoundsException

@Test
public void parseFaultyXml() {
    String xml = "<?xml version='1.0'><val>One</val>";
    Document doc = Jsoup.parse(xml, "", Parser.xmlParser());
}
Results in:
java.lang.IndexOutOfBoundsException: Index: 0, Size: 0

	at java.util.ArrayList.rangeCheck(ArrayList.java:657)
	at java.util.ArrayList.get(ArrayList.java:433)
	at org.jsoup.nodes.Element.child(Element.java:254)
	at org.jsoup.parser.XmlTreeBuilder.insert(XmlTreeBuilder.java:91)
	at org.jsoup.parser.XmlTreeBuilder.process(XmlTreeBuilder.java:49)
	at org.jsoup.parser.TreeBuilder.runParser(TreeBuilder.java:52)
	at org.jsoup.parser.TreeBuilder.parse(TreeBuilder.java:45)
	at org.jsoup.parser.Parser.parseInput(Parser.java:34)
	at org.jsoup.Jsoup.parse(Jsoup.java:45)


public void head(org.jsoup.nodes.Node source, int depth) {
    namespacesStack.push(new HashMap<>(namespacesStack.peek())); // inherit from above on the stack
    if (source instanceof org.jsoup.nodes.Element) {
        org.jsoup.nodes.Element sourceEl = (org.jsoup.nodes.Element) source;

        String prefix = updateNamespaces(sourceEl);
        String namespace = namespacesStack.peek().get(prefix);
        String tagName = sourceEl.tagName();

        Element el =
            doc.createElementNS(namespace, tagName);
        copyAttributes(sourceEl, el);
        if (dest == null) { // sets up the root
            doc.appendChild(el);
        } else {
            dest.appendChild(el);
        }
        dest = el; // descend
    } else if (source instanceof org.jsoup.nodes.TextNode) {
        org.jsoup.nodes.TextNode sourceText = (org.jsoup.nodes.TextNode) source;
        Text text = doc.createTextNode(sourceText.getWholeText());
        dest.appendChild(text);
    } else if (source instanceof org.jsoup.nodes.Comment) {
        org.jsoup.nodes.Comment sourceComment = (org.jsoup.nodes.Comment) source;
        Comment comment = doc.createComment(sourceComment.getData());
        dest.appendChild(comment);
    } else if (source instanceof org.jsoup.nodes.DataNode) {
        org.jsoup.nodes.DataNode sourceData = (org.jsoup.nodes.DataNode) source;
        Text node = doc.createTextNode(sourceData.getWholeData());
        dest.appendChild(node);
    } else {
        // unhandled
    }
}

public void head(org.jsoup.nodes.Node source, int depth) {
    namespacesStack.push(new HashMap<>(namespacesStack.peek())); // inherit from above on the stack
    if (source instanceof org.jsoup.nodes.Element) {
        org.jsoup.nodes.Element sourceEl = (org.jsoup.nodes.Element) source;

        String prefix = updateNamespaces(sourceEl);
        String namespace = namespacesStack.peek().get(prefix);
        String tagName = sourceEl.tagName();

        Element el = namespace == null && tagName.contains(":") ?
            doc.createElementNS("", tagName) : // doesn't have a real namespace defined
            doc.createElementNS(namespace, tagName);
        copyAttributes(sourceEl, el);
        if (dest == null) { // sets up the root
            doc.appendChild(el);
        } else {
            dest.appendChild(el);
        }
        dest = el; // descend
    } else if (source instanceof org.jsoup.nodes.TextNode) {
        org.jsoup.nodes.TextNode sourceText = (org.jsoup.nodes.TextNode) source;
        Text text = doc.createTextNode(sourceText.getWholeText());
        dest.appendChild(text);
    } else if (source instanceof org.jsoup.nodes.Comment) {
        org.jsoup.nodes.Comment sourceComment = (org.jsoup.nodes.Comment) source;
        Comment comment = doc.createComment(sourceComment.getData());
        dest.appendChild(comment);
    } else if (source instanceof org.jsoup.nodes.DataNode) {
        org.jsoup.nodes.DataNode sourceData = (org.jsoup.nodes.DataNode) source;
        Text node = doc.createTextNode(sourceData.getWholeData());
        dest.appendChild(node);
    } else {
        // unhandled
    }
}

W3CDom Helper fails to convert whenever some namespace declarations are missing

Hello
I've been running into an issue where if I convert my Jsoup parsed document into a org.w3c.dom.Document with the W3CDom helper and that document happens to be missing namespace declarations we get the following exception:
NAMESPACE_ERR: An attempt is made to create or change an object in a way which is incorrect with regard to namespaces.

I've looked into this a bit and first thing I tried was using a locally forked version of the W3CDom helper that simply turned this flag off:
factory.setNamespaceAware(false);

However the issue continued, so instead I simply hacked the code to completely ignore namespaces
// (csueiras): We purposely remove any namespace because we get malformed HTML that might not be
// declaring all of it's namespaces!
Element el = doc.createElementNS("", sourceEl.tagName());

I am not completely sure if this will have any side effects, but it resolved the issues with the document I'm interacting with. I would be glad to provide a pull request if I have some guidance regarding how to properly handle this issue if it can be handled by Jsoup.
The document I'm having issues is simply making use of the Facebook like buttons using tags like this:
<fb:like ...

But there's no namespace declaration for "fb".


public Attribute(String key, String val, Attributes parent) {
    Validate.notNull(key);
    this.key = key.trim();
    Validate.notEmpty(key); // trimming could potentially make empty, so validate here
    this.val = val;
    this.parent = parent;
}

public Attribute(String key, String val, Attributes parent) {
    Validate.notNull(key);
    key = key.trim();
    Validate.notEmpty(key); // trimming could potentially make empty, so validate here
    this.key = key;
    this.val = val;
    this.parent = parent;
}

Attribute.java line 45 variable key scope error, it seems should be "this.key"

Attribute.java Line 45, it should be:
Validate.notEmpty(this.key);
rather than
Validate.notEmpty(key);
This issue only happens when key is blank or empty, in reality this would rarely happen, but in the syntax context it is still an issue, so better fix this.


public XmlDeclaration asXmlDeclaration() {
    String data = getData();
    Document doc = Jsoup.parse("<" + data.substring(1, data.length() -1) + ">", baseUri(), Parser.xmlParser());
    XmlDeclaration decl = null;
    if (doc.childNodeSize() > 0) {
        Element el = doc.child(0);
        decl = new XmlDeclaration(NodeUtils.parser(doc).settings().normalizeTag(el.tagName()), data.startsWith("!"));
        decl.attributes().addAll(el.attributes());
    }
    return decl;
}

public XmlDeclaration asXmlDeclaration() {
    String data = getData();
    Document doc = Jsoup.parse("<" + data.substring(1, data.length() -1) + ">", baseUri(), Parser.xmlParser());
    XmlDeclaration decl = null;
    if (doc.children().size() > 0) {
        Element el = doc.child(0);
        decl = new XmlDeclaration(NodeUtils.parser(doc).settings().normalizeTag(el.tagName()), data.startsWith("!"));
        decl.attributes().addAll(el.attributes());
    }
    return decl;
}

Hi, I am using Jsoup 1.11.3. While trying to parse HTML content, I'm getting IndexOutOfBoundsException.
I am using such Jsoup call as this is the only way to parse iframe content.
Jsoup call:
Jsoup.parse(html, "", Parser.xmlParser())
HTML is here:  https://files.fm/u/v43yemgb. I can't add it to the body as it's huge.


public String getValue() {
    return val;
}

public String getValue() {
    return Attributes.checkNotNull(val);
}

Attribute.getValue() broken for empty attributes since 1.11.1

Document doc = Jsoup.parse("<div hidden>");
        Attributes attributes = doc.body().child(0).attributes();
        System.out.println(String.format("Attr: '%s', value: '%s'", "hidden",
                attributes.get("hidden")));

        Attribute first = attributes.iterator().next();
        System.out.println(String.format("Attr: '%s', value: '%s'",
                first.getKey(), first.getValue()));

Expected output, as in 1.10.x
Attr: 'hidden', value: ''
Attr: 'hidden', value: ''

Output in 1.11.1-1.11.3:
Attr: 'hidden', value: ''
Attr: 'hidden', value: 'null'


public String setValue(String val) {
    String oldVal = parent.get(this.key);
    if (parent != null) {
        int i = parent.indexOfKey(this.key);
        if (i != Attributes.NotFound)
            parent.vals[i] = val;
    }
    this.val = val;
    return Attributes.checkNotNull(oldVal);
}

public String setValue(String val) {
    String oldVal = this.val;
    if (parent != null) {
        oldVal = parent.get(this.key); // trust the container more
        int i = parent.indexOfKey(this.key);
        if (i != Attributes.NotFound)
            parent.vals[i] = val;
    }
    this.val = val;
    return Attributes.checkNotNull(oldVal);
}

NPE in Attribute.setValue() for attribute without parent

public String setValue(String val) {
        String oldVal = parent.get(this.key);
        if (parent != null) {
            int i = parent.indexOfKey(this.key);
            if (i != Attributes.NotFound)
                parent.vals[i] = val;
        }
        this.val = val;
        return oldVal;
    }

Its useless to check parent for null after it has been dereferenced. I guess this is a copy-paste-bug:
    public void setKey(String key) {
        Validate.notNull(key);
        key = key.trim();
        Validate.notEmpty(key); // trimming could potentially make empty, so validate here
        if (parent != null) {
            int i = parent.indexOfKey(this.key);
            if (i != Attributes.NotFound)
                parent.keys[i] = key;
        }
        this.key = key;
    }


private static boolean looksLikeUtf8(byte[] input) {
    int i = 0;
    // BOM:
    if (input.length >= 3 && (input[0] & 0xFF) == 0xEF
        && (input[1] & 0xFF) == 0xBB & (input[2] & 0xFF) == 0xBF) {
        i = 3;
    }

    int end;
    for (int j = input.length; i < j; ++i) {
        int o = input[i];
        if ((o & 0x80) == 0) {
            continue; // ASCII
        }

        // UTF-8 leading:
        if ((o & 0xE0) == 0xC0) {
            end = i + 1;
        } else if ((o & 0xF0) == 0xE0) {
            end = i + 2;
        } else if ((o & 0xF8) == 0xF0) {
            end = i + 3;
        } else {
            return false;
        }

        while (i < end) {
            i++;
            o = input[i];
            if ((o & 0xC0) != 0x80) {
                return false;
            }
        }
    }
    return true;
}

private static boolean looksLikeUtf8(byte[] input) {
    int i = 0;
    // BOM:
    if (input.length >= 3 && (input[0] & 0xFF) == 0xEF
        && (input[1] & 0xFF) == 0xBB & (input[2] & 0xFF) == 0xBF) {
        i = 3;
    }

    int end;
    for (int j = input.length; i < j; ++i) {
        int o = input[i];
        if ((o & 0x80) == 0) {
            continue; // ASCII
        }

        // UTF-8 leading:
        if ((o & 0xE0) == 0xC0) {
            end = i + 1;
        } else if ((o & 0xF0) == 0xE0) {
            end = i + 2;
        } else if ((o & 0xF8) == 0xF0) {
            end = i + 3;
        } else {
            return false;
        }

        if (end >= input.length)
            return false;

        while (i < end) {
            i++;
            o = input[i];
            if ((o & 0xC0) != 0x80) {
                return false;
            }
        }
    }
    return true;
}

ArrayIndexOutOfBoundsException when parsing with some URL

error
Caused by: java.lang.ArrayIndexOutOfBoundsException: 11
	at org.jsoup.helper.HttpConnection$Base.looksLikeUtf8(HttpConnection.java:437)
	at org.jsoup.helper.HttpConnection$Base.fixHeaderEncoding(HttpConnection.java:400)
	at org.jsoup.helper.HttpConnection$Base.addHeader(HttpConnection.java:386)
	at org.jsoup.helper.HttpConnection$Response.processResponseHeaders(HttpConnection.java:1075)
	at org.jsoup.helper.HttpConnection$Response.setupFromConnection(HttpConnection.java:1019)
	at org.jsoup.helper.HttpConnection$Response.execute(HttpConnection.java:752)
	at org.jsoup.helper.HttpConnection$Response.execute(HttpConnection.java:722)
	at org.jsoup.helper.HttpConnection.execute(HttpConnection.java:306)

code
try {
            String url = "https://www.colisprive.com/moncolis/pages/detailColis.aspx?numColis=P4000000037777930";
            Connection connection = Jsoup.connect(url).referrer(url).
                    userAgent("Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36")
                    .ignoreContentType(true).timeout(20000);

            connection.method(Method.GET);
            return connection.execute().parse();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }


public List<Connection.KeyVal> formData() {
    ArrayList<Connection.KeyVal> data = new ArrayList<>();

    // iterate the form control elements and accumulate their values
    for (Element el: elements) {
        if (!el.tag().isFormSubmittable()) continue; // contents are form listable, superset of submitable
        if (el.hasAttr("disabled")) continue; // skip disabled form inputs
        String name = el.attr("name");
        if (name.length() == 0) continue;
        String type = el.attr("type");

        if ("select".equals(el.normalName())) {
            Elements options = el.select("option[selected]");
            boolean set = false;
            for (Element option: options) {
                data.add(HttpConnection.KeyVal.create(name, option.val()));
                set = true;
            }
            if (!set) {
                Element option = el.select("option").first();
                if (option != null)
                    data.add(HttpConnection.KeyVal.create(name, option.val()));
            }
        } else if ("checkbox".equalsIgnoreCase(type) || "radio".equalsIgnoreCase(type)) {
            // only add checkbox or radio if they have the checked attribute
            if (el.hasAttr("checked")) {
                final String val = el.val().length() >  0 ? el.val() : "on";
                data.add(HttpConnection.KeyVal.create(name, val));
            }
        } else {
            data.add(HttpConnection.KeyVal.create(name, el.val()));
        }
    }
    return data;
}

public List<Connection.KeyVal> formData() {
    ArrayList<Connection.KeyVal> data = new ArrayList<>();

    // iterate the form control elements and accumulate their values
    for (Element el: elements) {
        if (!el.tag().isFormSubmittable()) continue; // contents are form listable, superset of submitable
        if (el.hasAttr("disabled")) continue; // skip disabled form inputs
        String name = el.attr("name");
        if (name.length() == 0) continue;
        String type = el.attr("type");

        if (type.equalsIgnoreCase("button")) continue; // browsers don't submit these

        if ("select".equals(el.normalName())) {
            Elements options = el.select("option[selected]");
            boolean set = false;
            for (Element option: options) {
                data.add(HttpConnection.KeyVal.create(name, option.val()));
                set = true;
            }
            if (!set) {
                Element option = el.select("option").first();
                if (option != null)
                    data.add(HttpConnection.KeyVal.create(name, option.val()));
            }
        } else if ("checkbox".equalsIgnoreCase(type) || "radio".equalsIgnoreCase(type)) {
            // only add checkbox or radio if they have the checked attribute
            if (el.hasAttr("checked")) {
                final String val = el.val().length() >  0 ? el.val() : "on";
                data.add(HttpConnection.KeyVal.create(name, val));
            }
        } else {
            data.add(HttpConnection.KeyVal.create(name, el.val()));
        }
    }
    return data;
}

<input type="image"> is not special cased in formData method

The following code:
import org.jsoup.Jsoup;
import org.jsoup.nodes.FormElement;

class Scratch {
    public static void main(String[] args) {
        System.out.println(((FormElement) Jsoup.parse("<form id=f><input type=image name=x></form>").getElementById("f")).formData());
    }
}
Returns the following output:
[x=]

When either [] or [x.x=0, x.y=0] is expected (not sure which, but [x=] is definitely wrong).


public final Object computeValue(EvalContext context) {
    return compute(args[0].computeValue(context), args[1].computeValue(context))
            ? Boolean.TRUE : Boolean.FALSE;
}

public final Object computeValue(EvalContext context) {
    return compute(args[0].compute(context), args[1].compute(context))
            ? Boolean.TRUE : Boolean.FALSE;
}

Binary operators behaviour involving node-sets is incorrect

According to XPath specification:
"If both objects to be compared are node-sets, then the comparison will be true if and only if there is a node in the first node-set and a node in the second node-set such that the result of performing the comparison on the string-values of the two nodes is true. If one object to be compared is a node-set and the other is a number, then the comparison will be true if and only if there is a node in the node-set such that the result of performing the comparison on the number to be compared and on the result of converting the string-value of that node to a number using the number function is true."
But following example illustrates, that this is not a JXPath behaviour:
        JXPathContext pathContext = JXPathContext
                .newContext(DocumentBuilderFactory.newInstance()
                        .newDocumentBuilder().parse(
                                new InputSource(new StringReader(
                                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n"
                                                + "<doc/>"))));
        Boolean result = (Boolean) pathContext.getValue("2.0 > child1",
                Boolean.class);
        assertFalse(result.booleanValue());
"child1" is not found - right operand node set is empty, but result is TRUE, instead of FALSE.
Please, check greaterThan(), lesserThan(), etc methods of org.apache.xpath.objects.XObject for possible solution


public static boolean testNode(Node node, NodeTest test) {
    if (test == null) {
        return true;
    }
    if (test instanceof NodeNameTest) {
        if (node.getNodeType() != Node.ELEMENT_NODE) {
            return false;
        }

        NodeNameTest nodeNameTest = (NodeNameTest) test;
        QName testName = nodeNameTest.getNodeName();
        String namespaceURI = nodeNameTest.getNamespaceURI();
        boolean wildcard = nodeNameTest.isWildcard();
        String testPrefix = testName.getPrefix();
        if (wildcard && testPrefix == null) {
            return true;
        }
        if (wildcard
            || testName.getName()
                    .equals(DOMNodePointer.getLocalName(node))) {
            String nodeNS = DOMNodePointer.getNamespaceURI(node);
            return equalStrings(namespaceURI, nodeNS);
        }
        return false;
    }
    if (test instanceof NodeTypeTest) {
        int nodeType = node.getNodeType();
        switch (((NodeTypeTest) test).getNodeType()) {
            case Compiler.NODE_TYPE_NODE :
                return nodeType == Node.ELEMENT_NODE
                        || nodeType == Node.DOCUMENT_NODE;
            case Compiler.NODE_TYPE_TEXT :
                return nodeType == Node.CDATA_SECTION_NODE
                    || nodeType == Node.TEXT_NODE;
            case Compiler.NODE_TYPE_COMMENT :
                return nodeType == Node.COMMENT_NODE;
            case Compiler.NODE_TYPE_PI :
                return nodeType == Node.PROCESSING_INSTRUCTION_NODE;
        }
        return false;
    }
    if (test instanceof ProcessingInstructionTest) {
        if (node.getNodeType() == Node.PROCESSING_INSTRUCTION_NODE) {
            String testPI = ((ProcessingInstructionTest) test).getTarget();
            String nodePI = ((ProcessingInstruction) node).getTarget();
            return testPI.equals(nodePI);
        }
    }
    return false;
}

public static boolean testNode(Node node, NodeTest test) {
    if (test == null) {
        return true;
    }
    if (test instanceof NodeNameTest) {
        if (node.getNodeType() != Node.ELEMENT_NODE) {
            return false;
        }

        NodeNameTest nodeNameTest = (NodeNameTest) test;
        QName testName = nodeNameTest.getNodeName();
        String namespaceURI = nodeNameTest.getNamespaceURI();
        boolean wildcard = nodeNameTest.isWildcard();
        String testPrefix = testName.getPrefix();
        if (wildcard && testPrefix == null) {
            return true;
        }
        if (wildcard
            || testName.getName()
                    .equals(DOMNodePointer.getLocalName(node))) {
            String nodeNS = DOMNodePointer.getNamespaceURI(node);
            return equalStrings(namespaceURI, nodeNS) || nodeNS == null
                    && equalStrings(testPrefix, getPrefix(node));
        }
        return false;
    }
    if (test instanceof NodeTypeTest) {
        int nodeType = node.getNodeType();
        switch (((NodeTypeTest) test).getNodeType()) {
            case Compiler.NODE_TYPE_NODE :
                return nodeType == Node.ELEMENT_NODE
                        || nodeType == Node.DOCUMENT_NODE;
            case Compiler.NODE_TYPE_TEXT :
                return nodeType == Node.CDATA_SECTION_NODE
                    || nodeType == Node.TEXT_NODE;
            case Compiler.NODE_TYPE_COMMENT :
                return nodeType == Node.COMMENT_NODE;
            case Compiler.NODE_TYPE_PI :
                return nodeType == Node.PROCESSING_INSTRUCTION_NODE;
        }
        return false;
    }
    if (test instanceof ProcessingInstructionTest) {
        if (node.getNodeType() == Node.PROCESSING_INSTRUCTION_NODE) {
            String testPI = ((ProcessingInstructionTest) test).getTarget();
            String nodePI = ((ProcessingInstruction) node).getTarget();
            return testPI.equals(nodePI);
        }
    }
    return false;
}

Incomplete handling of undefined namespaces

Mcduffey, Joe <jdmcduf@nsa.gov>
Can someone tell me how to register namespaces so that attributes with namespaces does not cause the exception
org.apache.common.ri.model.dom.DOMNodePointer.createAttribute
unknown namespace prefix: xsi
For example the following
<ElementA  A:myAttr="Mytype">
  <B:ElementB>MY VALUE</B:ElementB>
</ElementA>
Would result in the following exception:
org.apache.common.ri.model.dom.DOMNodePointer.createAttribute
unknown namespace prefix: A
FYI: In this example there was a namespace decaration in the file and I also manually called the
registerNamespace(A,"/http...");
registerNamespace(B,"/http...");
There was no problem encountered for elements. Only attributes. Can someone help? Thanks.


public boolean setPosition(int position) {
        if (!prepared) {
            prepared = true;
            BasicNodeSet nodeSet = (BasicNodeSet) getNodeSet();
            ArrayList pointers = new ArrayList();
            for (int i = 0; i < contexts.length; i++) {
                EvalContext ctx = (EvalContext) contexts[i];
                while (ctx.nextSet()) {
                    while (ctx.nextNode()) {
                        NodePointer ptr = ctx.getCurrentNodePointer();
                        if (!pointers.contains(ptr)) {
                            nodeSet.add(ptr);
                            pointers.add(ptr);
                        }
                    }
                }
            }
        }
        return super.setPosition(position);
    }

public boolean setPosition(int position) {
        if (!prepared) {
            prepared = true;
            BasicNodeSet nodeSet = (BasicNodeSet) getNodeSet();
            ArrayList pointers = new ArrayList();
            for (int i = 0; i < contexts.length; i++) {
                EvalContext ctx = (EvalContext) contexts[i];
                while (ctx.nextSet()) {
                    while (ctx.nextNode()) {
                        NodePointer ptr = ctx.getCurrentNodePointer();
                        if (!pointers.contains(ptr)) {
                            pointers.add(ptr);
                        }
                    }
                }
            }
            sortPointers(pointers);

            for (Iterator it = pointers.iterator(); it.hasNext();) {
                nodeSet.add((Pointer) it.next());
            }
        }
        return super.setPosition(position);
    }

Core union operation does not sort result nodes according to document order

Source document:
avaluebvalue

According to string() function defintion:
"A node-set is converted to a string by returning the string-value of the node in the node-set that is first in document order. If the node-set is empty, an empty string is returned."

Following XPath calculated incorrectly:
 string(/MAIN/B | /MAIN/A)

Expected result: "avalue"
Actual value: "bvalue"

Reason:
sorting of result nodes is missing from CoreOperationUnion


public boolean nextNode() {
        super.setPosition(getCurrentPosition() + 1);
        if (!setStarted) {
            setStarted = true;
            if (!(nodeTest instanceof NodeNameTest)) {
                return false;
            }
            QName name = ((NodeNameTest) nodeTest).getNodeName();
            iterator =
                parentContext.getCurrentNodePointer().attributeIterator(name);
        }
        if (iterator == null) {
            return false;
        }
        if (!iterator.setPosition(iterator.getPosition() + 1)) {
            return false;
        }
        currentNodePointer = iterator.getNodePointer();
        return true;
    }

public boolean nextNode() {
        super.setPosition(getCurrentPosition() + 1);
        if (!setStarted) {
            setStarted = true;
            NodeNameTest nodeNameTest = null;
            if (nodeTest instanceof NodeTypeTest) {
                if (((NodeTypeTest) nodeTest).getNodeType() == Compiler.NODE_TYPE_NODE) {
                    nodeNameTest = WILDCARD_TEST;
                }
            }
            else if (nodeTest instanceof NodeNameTest) {
                nodeNameTest = (NodeNameTest) nodeTest;
            }
            if (nodeNameTest == null) {
                return false;
            }
            iterator = parentContext.getCurrentNodePointer().attributeIterator(
                    nodeNameTest.getNodeName());
        }
        if (iterator == null) {
            return false;
        }
        if (!iterator.setPosition(iterator.getPosition() + 1)) {
            return false;
        }
        currentNodePointer = iterator.getNodePointer();
        return true;
    }

Checking test (Issue172_CountAttributeNode) I came with the following fix for the code in AttributeContext  line 72
from
-----
if (!(nodeTest instanceof NodeNameTest)) {
                return false;
            }
            QName name = ((NodeNameTest) nodeTest).getNodeName();

------
'
to
--- (outside method)
private static final QName WILDCARD = new QName("", "*");
--- (in method)

final QName name ;
if (nodeTest instanceof NodeTypeTest)
{
	 if (((NodeTypeTest) nodeTest).getNodeType() == Compiler.NODE_TYPE_NODE)
		 name = WILDCARD;
	 else return false;
}
else if (nodeTest instanceof NodeNameTest) {
	name = ((NodeNameTest) nodeTest).getNodeName();
}
else
{
	return false;
}


private boolean compute(Object left, Object right) {
        left = reduce(left);
        right = reduce(right);

        if (left instanceof InitialContext) {
            ((InitialContext) left).reset();
        }
        if (right instanceof InitialContext) {
            ((InitialContext) right).reset();
        }
        if (left instanceof Iterator && right instanceof Iterator) {
            return findMatch((Iterator) left, (Iterator) right);
        }
        if (left instanceof Iterator) {
            return containsMatch((Iterator) left, right);
        }
        if (right instanceof Iterator) {
            return containsMatch((Iterator) right, left);
        }
        double ld = InfoSetUtil.doubleValue(left);
        if (Double.isNaN(ld)) {
            return false;
        }
        double rd = InfoSetUtil.doubleValue(right);
        if (Double.isNaN(rd)) {
            return false;
        }
        return evaluateCompare(ld == rd ? 0 : ld < rd ? -1 : 1);
    }

private boolean compute(Object left, Object right) {
        left = reduce(left);
        right = reduce(right);

        if (left instanceof InitialContext) {
            ((InitialContext) left).reset();
        }
        if (right instanceof InitialContext) {
            ((InitialContext) right).reset();
        }
        if (left instanceof Iterator && right instanceof Iterator) {
            return findMatch((Iterator) left, (Iterator) right);
        }
        if (left instanceof Iterator) {
            return containsMatch((Iterator) left, right);
        }
        if (right instanceof Iterator) {
            return containsMatch(left, (Iterator) right);
        }
        double ld = InfoSetUtil.doubleValue(left);
        if (Double.isNaN(ld)) {
            return false;
        }
        double rd = InfoSetUtil.doubleValue(right);
        if (Double.isNaN(rd)) {
            return false;
        }
        return evaluateCompare(ld == rd ? 0 : ld < rd ? -1 : 1);
    }

relational operations do not function properly when comparing a non-Iterator LHS to an Iterator RHS

I have a simple JXpathContext, with the following variables: var1=0, var2=0, var3=1. When I try to evaluate the following expression - "$var1 + $var2 <= $var3", it returns false.


public int getLength() {
    return ValueUtils.getLength(getBaseValue());
}

public int getLength() {
    Object baseValue = getBaseValue();
    return baseValue == null ? 1 : ValueUtils.getLength(baseValue);
}

Comparing a vaule to null using unequals (!=) yields false!

        Map<String, Integer> m = new HashMap<String, Integer>();
        m.put("a", 1);
        m.put("b", null);
        m.put("c", 1);
        JXPathContext c = JXPathContext.newContext(m);
        System.out.println(c.getValue("a != b") + " should be true");
        System.out.println(c.getValue("a != c") + " should be false");
        System.out.println(c.getValue("a = b") + " should be false");
        System.out.println(c.getValue("a = c") + " should be true");
        System.out.println(c.getValue("not(a = b)") + " should be true");
        System.out.println(c.getValue("not(a = c)") + " should be false");

Output using 1.3:
 false should be true
false should be false
false should be false
true should be true
true should be true
false should be false
In 1.2 it works correctly!


public static String getNamespaceURI(Node node) {
    if (node instanceof Document) {
        node = ((Document) node).getDocumentElement();
    }

    Element element = (Element) node;

    String uri = element.getNamespaceURI();
    if (uri == null) {
        String prefix = getPrefix(node);
        String qname = prefix == null ? "xmlns" : "xmlns:" + prefix;

        Node aNode = node;
        while (aNode != null) {
            if (aNode.getNodeType() == Node.ELEMENT_NODE) {
                Attr attr = ((Element) aNode).getAttributeNode(qname);
                if (attr != null) {
                    return attr.getValue();
                }
            }
            aNode = aNode.getParentNode();
        }
        return null;
    }
    return uri;
}

public static String getNamespaceURI(Node node) {
    if (node instanceof Document) {
        node = ((Document) node).getDocumentElement();
    }

    Element element = (Element) node;

    String uri = element.getNamespaceURI();
    if (uri == null) {
        String prefix = getPrefix(node);
        String qname = prefix == null ? "xmlns" : "xmlns:" + prefix;

        Node aNode = node;
        while (aNode != null) {
            if (aNode.getNodeType() == Node.ELEMENT_NODE) {
                Attr attr = ((Element) aNode).getAttributeNode(qname);
                if (attr != null) {
                    uri = attr.getValue();
                    break;
                }
            }
            aNode = aNode.getParentNode();
        }
    }
    return "".equals(uri) ? null : uri;
}

Resetting the default namespace causes a serious endless loop when requesting .asPath() on a node.

sample smaller case:

<...>
 <b:foo xmlns:b="bla" xmlns="test111">    <!--  No nodes are placed in the tree within ns "test111" but the attribute is still there.-->
  <b:bar>a</b:bar>                         <!-- is in ns 'bla' -->
  <test xmlns=""></test>                   <!-- does not have a namespace -->
 </b:foo>
</...>

when requesting .asPath() on the 'test' node, it loops in org.apache.commons.jxpath.ri.NamespaceResolver.getPrefix(NodePointer, String),
and if it didn't loop it would create a wrong xpath '//b:fo/null:test' DOMNodePointer.asPath().
So I think that the fix should be in org.apache.commons.jxpath.ri.model.dom.DOMNodePointer.asPath()

....
                    String ln = DOMNodePointer.getLocalName(node);
                    String nsURI = getNamespaceURI();
                    if (nsURI == null) {
                        buffer.append(ln);
                        buffer.append('[');
                        buffer.append(getRelativePositionByName()).append(']');
                    }
                    else {
                        String prefix = getNamespaceResolver().getPrefix(nsURI);
                        if (prefix != null) {
...

should become

...
                    String ln = DOMNodePointer.getLocalName(node);
                    String nsURI = getNamespaceURI();
                    if (nsURI == null || nsURI.length() == 0) { // check for empty string which means that the node doesn't have a namespace.
                        buffer.append(ln);
                        buffer.append('[');
                        buffer.append(getRelativePositionByName()).append(']');
                    }
                    else {
                        String prefix = getNamespaceResolver().getPrefix(nsURI);
                        if (prefix != null) {
...


private int compareNodePointers(
    NodePointer p1,
    int depth1,
    NodePointer p2,
    int depth2)
{
    if (depth1 < depth2) {
        int r = compareNodePointers(p1, depth1, p2.parent, depth2 - 1);
        return r == 0 ? -1 : r;
    }
    if (depth1 > depth2) {
        int r = compareNodePointers(p1.parent, depth1 - 1, p2, depth2);
        return r == 0 ? 1 : r;
    }
    if (p1 == null && p2 == null) {
        return 0;
    }

    if (p1 != null && p1.equals(p2)) {
        return 0;
    }

    if (depth1 == 1) {
        throw new JXPathException(
                "Cannot compare pointers that do not belong to the same tree: '"
                        + p1 + "' and '" + p2 + "'");
    }
    int r = compareNodePointers(p1.parent, depth1 - 1, p2.parent, depth2 - 1);
    if (r != 0) {
        return r;
    }

    return p1.parent.compareChildNodePointers(p1, p2);
}

private int compareNodePointers(
    NodePointer p1,
    int depth1,
    NodePointer p2,
    int depth2)
{
    if (depth1 < depth2) {
        int r = compareNodePointers(p1, depth1, p2.parent, depth2 - 1);
        return r == 0 ? -1 : r;
    }
    if (depth1 > depth2) {
        int r = compareNodePointers(p1.parent, depth1 - 1, p2, depth2);
        return r == 0 ? 1 : r;
    }
    if (p1 == null && p2 == null) {
        return 0;
    }

    if (p1 != null && p1.equals(p2)) {
        return 0;
    }

    if (depth1 == 1) {
        return 0;
    }
    int r = compareNodePointers(p1.parent, depth1 - 1, p2.parent, depth2 - 1);
    if (r != 0) {
        return r;
    }

    return p1.parent.compareChildNodePointers(p1, p2);
}

For XPath "$var | /MAIN/A" exception is thrown:
org.apache.commons.jxpath.JXPathException: Cannot compare pointers that do not belong to the same tree: '$var' and ''
	at org.apache.commons.jxpath.ri.model.NodePointer.compareNodePointers(NodePointer.java:665)
	at org.apache.commons.jxpath.ri.model.NodePointer.compareNodePointers(NodePointer.java:649)
	at org.apache.commons.jxpath.ri.model.NodePointer.compareNodePointers(NodePointer.java:649)
	at org.apache.commons.jxpath.ri.model.NodePointer.compareTo(NodePointer.java:639)
	at java.util.Arrays.mergeSort(Arrays.java:1152)
	at java.util.Arrays.sort(Arrays.java:1079)
	at java.util.Collections.sort(Collections.java:113)
	at org.apache.commons.jxpath.ri.EvalContext.constructIterator(EvalContext.java:176)
	at org.apache.commons.jxpath.ri.EvalContext.hasNext(EvalContext.java:100)
	at org.apache.commons.jxpath.JXPathContext.selectNodes(JXPathContext.java:648)
	at org.apache.commons.jxpath.ri.model.VariablePointerTestCase.testUnionOfVariableAndNode(VariablePointerTestCase.java:76)


protected boolean equal(
    EvalContext context,
    Expression left,
    Expression right)
{
    Object l = left.compute(context);
    Object r = right.compute(context);

      System.err.println("COMPARING: " +
          (l == null ? "null" : l.getClass().getName()) + " " +
          (r == null ? "null" : r.getClass().getName()));

    if (l instanceof InitialContext || l instanceof SelfContext) {
        l = ((EvalContext) l).getSingleNodePointer();
    }

    if (r instanceof InitialContext || r instanceof SelfContext) {
        r = ((EvalContext) r).getSingleNodePointer();
    }

    if (l instanceof Collection) {
        l = ((Collection) l).iterator();
    }

    if (r instanceof Collection) {
        r = ((Collection) r).iterator();
    }

    if ((l instanceof Iterator) && !(r instanceof Iterator)) {
        return contains((Iterator) l, r);
    }
    if (!(l instanceof Iterator) && (r instanceof Iterator)) {
        return contains((Iterator) r, l);
    }
    if (l instanceof Iterator && r instanceof Iterator) {
        return findMatch((Iterator) l, (Iterator) r);
    }
    return equal(l, r);
}

protected boolean equal(
    EvalContext context,
    Expression left,
    Expression right)
{
    Object l = left.compute(context);
    Object r = right.compute(context);

      System.err.println("COMPARING: " +
          (l == null ? "null" : l.getClass().getName()) + " " +
          (r == null ? "null" : r.getClass().getName()));

    if (l instanceof InitialContext) {
        ((EvalContext) l).reset();
    }

    if (l instanceof SelfContext) {
        l = ((EvalContext) l).getSingleNodePointer();
    }

    if (r instanceof InitialContext) {
        ((EvalContext) r).reset();
    }

    if (r instanceof SelfContext) {
        r = ((EvalContext) r).getSingleNodePointer();
    }

    if (l instanceof Collection) {
        l = ((Collection) l).iterator();
    }

    if (r instanceof Collection) {
        r = ((Collection) r).iterator();
    }

    if ((l instanceof Iterator) && !(r instanceof Iterator)) {
        return contains((Iterator) l, r);
    }
    if (!(l instanceof Iterator) && (r instanceof Iterator)) {
        return contains((Iterator) r, l);
    }
    if (l instanceof Iterator && r instanceof Iterator) {
        return findMatch((Iterator) l, (Iterator) r);
    }
    return equal(l, r);
}

equality test for multi-valued variables does not conform to spec

given e.g. variable d=
{"a", "b"}
, the spec implies that "$d = 'a'" and that "$d = 'b'".  Instead of iterating the variable's components its immediate content (here, the String[]) is compared, causing the aforementioned assertions to fail.


private boolean compute(Object left, Object right) {
    left = reduce(left);
    right = reduce(right);

    if (left instanceof InitialContext) {
        ((InitialContext) left).reset();
    }
    if (right instanceof InitialContext) {
        ((InitialContext) right).reset();
    }
    if (left instanceof Iterator && right instanceof Iterator) {
        return findMatch((Iterator) left, (Iterator) right);
    }
    if (left instanceof Iterator) {
        return containsMatch((Iterator) left, right);
    }
    if (right instanceof Iterator) {
        return containsMatch((Iterator) right, left);
    }
    double ld = InfoSetUtil.doubleValue(left);
    double rd = InfoSetUtil.doubleValue(right);
    return evaluateCompare(ld == rd ? 0 : ld < rd ? -1 : 1);
}

private boolean compute(Object left, Object right) {
    left = reduce(left);
    right = reduce(right);

    if (left instanceof InitialContext) {
        ((InitialContext) left).reset();
    }
    if (right instanceof InitialContext) {
        ((InitialContext) right).reset();
    }
    if (left instanceof Iterator && right instanceof Iterator) {
        return findMatch((Iterator) left, (Iterator) right);
    }
    if (left instanceof Iterator) {
        return containsMatch((Iterator) left, right);
    }
    if (right instanceof Iterator) {
        return containsMatch((Iterator) right, left);
    }
    double ld = InfoSetUtil.doubleValue(left);
    if (Double.isNaN(ld)) {
        return false;
    }
    double rd = InfoSetUtil.doubleValue(right);
    if (Double.isNaN(rd)) {
        return false;
    }
    return evaluateCompare(ld == rd ? 0 : ld < rd ? -1 : 1);
}

'NaN' > 'NaN' is true, but should be FALSE


private static StringBuilder escapeRegex(StringBuilder regex, String value, boolean unquote) {
    boolean wasWhite= false;
    for(int i= 0; i<value.length(); ++i) {
        char c= value.charAt(i);
        if(Character.isWhitespace(c)) {
            if(!wasWhite) {
                wasWhite= true;
                regex.append("\\s*+");
            }
            continue;
        }
        wasWhite= false;
        switch(c) {
        case '\'':
            if(unquote) {
                if(++i==value.length()) {
                    return regex;
                }
                c= value.charAt(i);
            }
            break;
        case '?':
        case '[':
        case ']':
        case '(':
        case ')':
        case '{':
        case '}':
        case '\\':
        case '|':
        case '*':
        case '+':
        case '^':
        case '$':
        case '.':
            regex.append('\\');
        }
        regex.append(c);
    }
    return regex;
}

private static StringBuilder escapeRegex(StringBuilder regex, String value, boolean unquote) {
    for(int i= 0; i<value.length(); ++i) {
        char c= value.charAt(i);
        switch(c) {
        case '\'':
            if(unquote) {
                if(++i==value.length()) {
                    return regex;
                }
                c= value.charAt(i);
            }
            break;
        case '?':
        case '[':
        case ']':
        case '(':
        case ')':
        case '{':
        case '}':
        case '\\':
        case '|':
        case '*':
        case '+':
        case '^':
        case '$':
        case '.':
            regex.append('\\');
        }
        regex.append(c);
    }
    return regex;
}

FastDateParser does not handle white-space properly

The SimpleDateFormat Javadoc does not treat white-space specially, however FastDateParser treats a single white-space as being any number of white-space characters.
This means that FDP will parse dates that fail when parsed by SDP.


public static String random(int count, int start, int end, boolean letters, boolean numbers,
                            char[] chars, Random random) {
    if (count == 0) {
        return "";
    } else if (count < 0) {
        throw new IllegalArgumentException("Requested random string length " + count + " is less than 0.");
    }
    if (chars != null && chars.length == 0) {
        throw new IllegalArgumentException("The chars array must not be empty");
    }

    if (start == 0 && end == 0) {
        if (chars != null) {
            end = chars.length;
        } else {
            if (!letters && !numbers) {
                end = Integer.MAX_VALUE;
            } else {
                end = 'z' + 1;
                start = ' ';
            }
        }
    }

    char[] buffer = new char[count];
    int gap = end - start;

    while (count-- != 0) {
        char ch;
        if (chars == null) {
            ch = (char) (random.nextInt(gap) + start);
        } else {
            ch = chars[random.nextInt(gap) + start];
        }
        if (letters && Character.isLetter(ch)
                || numbers && Character.isDigit(ch)
                || !letters && !numbers) {
            if(ch >= 56320 && ch <= 57343) {
                if(count == 0) {
                    count++;
                } else {
                    // low surrogate, insert high surrogate after putting it in
                    buffer[count] = ch;
                    count--;
                    buffer[count] = (char) (55296 + random.nextInt(128));
                }
            } else if(ch >= 55296 && ch <= 56191) {
                if(count == 0) {
                    count++;
                } else {
                    // high surrogate, insert low surrogate before putting it in
                    buffer[count] = (char) (56320 + random.nextInt(128));
                    count--;
                    buffer[count] = ch;
                }
            } else if(ch >= 56192 && ch <= 56319) {
                // private high surrogate, no effing clue, so skip it
                count++;
            } else {
                buffer[count] = ch;
            }
        } else {
            count++;
        }
    }
    return new String(buffer);
}

public static String random(int count, int start, int end, boolean letters, boolean numbers,
                            char[] chars, Random random) {
    if (count == 0) {
        return "";
    } else if (count < 0) {
        throw new IllegalArgumentException("Requested random string length " + count + " is less than 0.");
    }
    if (chars != null && chars.length == 0) {
        throw new IllegalArgumentException("The chars array must not be empty");
    }

    if (start == 0 && end == 0) {
        if (chars != null) {
            end = chars.length;
        } else {
            if (!letters && !numbers) {
                end = Integer.MAX_VALUE;
            } else {
                end = 'z' + 1;
                start = ' ';
            }
        }
    } else {
        if (end <= start) {
            throw new IllegalArgumentException("Parameter end (" + end + ") must be greater than start (" + start + ")");
        }
    }

    char[] buffer = new char[count];
    int gap = end - start;

    while (count-- != 0) {
        char ch;
        if (chars == null) {
            ch = (char) (random.nextInt(gap) + start);
        } else {
            ch = chars[random.nextInt(gap) + start];
        }
        if (letters && Character.isLetter(ch)
                || numbers && Character.isDigit(ch)
                || !letters && !numbers) {
            if(ch >= 56320 && ch <= 57343) {
                if(count == 0) {
                    count++;
                } else {
                    // low surrogate, insert high surrogate after putting it in
                    buffer[count] = ch;
                    count--;
                    buffer[count] = (char) (55296 + random.nextInt(128));
                }
            } else if(ch >= 55296 && ch <= 56191) {
                if(count == 0) {
                    count++;
                } else {
                    // high surrogate, insert low surrogate before putting it in
                    buffer[count] = (char) (56320 + random.nextInt(128));
                    count--;
                    buffer[count] = ch;
                }
            } else if(ch >= 56192 && ch <= 56319) {
                // private high surrogate, no effing clue, so skip it
                count++;
            } else {
                buffer[count] = ch;
            }
        } else {
            count++;
        }
    }
    return new String(buffer);
}

RandomStringUtils throws confusing IAE when end <= start

RandomUtils invokes Random#nextInt where n = end - start.
If end <= start, then Random throws:
java.lang.IllegalArgumentException: n must be positive
This is confusing, and does not identify the source of the problem.


public static String random(int count, int start, int end, boolean letters, boolean numbers,
                            char[] chars, Random random) {
    if (count == 0) {
        return "";
    } else if (count < 0) {
        throw new IllegalArgumentException("Requested random string length " + count + " is less than 0.");
    }

    if (start == 0 && end == 0) {
            if (!letters && !numbers) {
                end = Integer.MAX_VALUE;
            } else {
                end = 'z' + 1;
                start = ' ';
            }
    }

    char[] buffer = new char[count];
    int gap = end - start;

    while (count-- != 0) {
        char ch;
        if (chars == null) {
            ch = (char) (random.nextInt(gap) + start);
        } else {
            ch = chars[random.nextInt(gap) + start];
        }
        if (letters && Character.isLetter(ch)
                || numbers && Character.isDigit(ch)
                || !letters && !numbers) {
            if(ch >= 56320 && ch <= 57343) {
                if(count == 0) {
                    count++;
                } else {
                    // low surrogate, insert high surrogate after putting it in
                    buffer[count] = ch;
                    count--;
                    buffer[count] = (char) (55296 + random.nextInt(128));
                }
            } else if(ch >= 55296 && ch <= 56191) {
                if(count == 0) {
                    count++;
                } else {
                    // high surrogate, insert low surrogate before putting it in
                    buffer[count] = (char) (56320 + random.nextInt(128));
                    count--;
                    buffer[count] = ch;
                }
            } else if(ch >= 56192 && ch <= 56319) {
                // private high surrogate, no effing clue, so skip it
                count++;
            } else {
                buffer[count] = ch;
            }
        } else {
            count++;
        }
    }
    return new String(buffer);
}

public static String random(int count, int start, int end, boolean letters, boolean numbers,
                            char[] chars, Random random) {
    if (count == 0) {
        return "";
    } else if (count < 0) {
        throw new IllegalArgumentException("Requested random string length " + count + " is less than 0.");
    }
    if (chars != null && chars.length == 0) {
        throw new IllegalArgumentException("The chars array must not be empty");
    }

    if (start == 0 && end == 0) {
        if (chars != null) {
            end = chars.length;
        } else {
            if (!letters && !numbers) {
                end = Integer.MAX_VALUE;
            } else {
                end = 'z' + 1;
                start = ' ';
            }
        }
    }

    char[] buffer = new char[count];
    int gap = end - start;

    while (count-- != 0) {
        char ch;
        if (chars == null) {
            ch = (char) (random.nextInt(gap) + start);
        } else {
            ch = chars[random.nextInt(gap) + start];
        }
        if (letters && Character.isLetter(ch)
                || numbers && Character.isDigit(ch)
                || !letters && !numbers) {
            if(ch >= 56320 && ch <= 57343) {
                if(count == 0) {
                    count++;
                } else {
                    // low surrogate, insert high surrogate after putting it in
                    buffer[count] = ch;
                    count--;
                    buffer[count] = (char) (55296 + random.nextInt(128));
                }
            } else if(ch >= 55296 && ch <= 56191) {
                if(count == 0) {
                    count++;
                } else {
                    // high surrogate, insert low surrogate before putting it in
                    buffer[count] = (char) (56320 + random.nextInt(128));
                    count--;
                    buffer[count] = ch;
                }
            } else if(ch >= 56192 && ch <= 56319) {
                // private high surrogate, no effing clue, so skip it
                count++;
            } else {
                buffer[count] = ch;
            }
        } else {
            count++;
        }
    }
    return new String(buffer);
}

RandomStringUtils.random(count, 0, 0, false, false, universe, random) always throws java.lang.ArrayIndexOutOfBoundsException

In commons-lang 2.6 line 250 :

ch = chars[random.nextInt(gap) + start];

This line of code takes a random int to fetch a char in the chars array regardless of its size.
(Besides start is useless here)
Fixed version would be :

//ch = chars[random.nextInt(gap)%chars.length];

When user pass 0 as end or when the array is not null but empty this line ends up with an exception


public static boolean equals(CharSequence cs1, CharSequence cs2) {
    if (cs1 == cs2) {
        return true;
    }
    if (cs1 == null || cs2 == null) {
        return false;
    }
        return cs1.equals(cs2);
}

public static boolean equals(CharSequence cs1, CharSequence cs2) {
    if (cs1 == cs2) {
        return true;
    }
    if (cs1 == null || cs2 == null) {
        return false;
    }
    if (cs1 instanceof String && cs2 instanceof String) {
        return cs1.equals(cs2);
    }
    return CharSequenceUtils.regionMatches(cs1, false, 0, cs2, 0, Math.max(cs1.length(), cs2.length()));
}

StringUtils equals() relies on undefined behavior

Since the java.lang.CharSequence class was first introduced in 1.4, the JavaDoc block has contained the following note:

This interface does not refine the general contracts of the equals and hashCode methods. The result of comparing two objects that implement CharSequence is therefore, in general, undefined. Each object may be implemented by a different class, and there is no guarantee that each class will be capable of testing its instances for equality with those of the other.
When the signature of the StringUtils equals() method was changed from equals(String, String) to equals(CharSequence, CharSequence) in R920543, the implementation still relied on calling CharSequence#equals(Object) even though, in general, the result is undefined.
One example where equals(Object) returns false even though, as CharSequences, two objects represent equal sequences is when one object is an instance of javax.lang.model.element.Name and the other object is a String.


public final void translate(CharSequence input, Writer out) throws IOException {
    if (out == null) {
        throw new IllegalArgumentException("The Writer must not be null");
    }
    if (input == null) {
        return;
    }
    int pos = 0;
    int len = Character.codePointCount(input, 0, input.length());
    while (pos < len) {
        int consumed = translate(input, pos, out);
        if (consumed == 0) {
            char[] c = Character.toChars(Character.codePointAt(input, pos));
            out.write(c);
        }
        else {
        // contract with translators is that they have to understand codepoints
        // and they just took care of a surrogate pair
        for (int pt = 0; pt < consumed; pt++) {
                if (pos < len - 2) {
            pos += Character.charCount(Character.codePointAt(input, pos));
                } else {
                    pos++;
                }
            }
            pos--;
        }
        pos++;
    }
}

public final void translate(CharSequence input, Writer out) throws IOException {
    if (out == null) {
        throw new IllegalArgumentException("The Writer must not be null");
    }
    if (input == null) {
        return;
    }
    int pos = 0;
    int len = input.length();
    while (pos < len) {
        int consumed = translate(input, pos, out);
        if (consumed == 0) {
            char[] c = Character.toChars(Character.codePointAt(input, pos));
            out.write(c);
            pos+= c.length;
            continue;
        }
        // contract with translators is that they have to understand codepoints
        // and they just took care of a surrogate pair
        for (int pt = 0; pt < consumed; pt++) {
            pos += Character.charCount(Character.codePointAt(input, pos));
        }
    }
}

StringEscapeUtils.escapeXml(input) outputs wrong results when an input contains characters in Supplementary Planes.

Hello.
I use StringEscapeUtils.escapeXml(input) to escape special characters for XML.
This method outputs wrong results when input contains characters in Supplementary Planes.
String str1 = "\uD842\uDFB7" + "A";
String str2 = StringEscapeUtils.escapeXml(str1);
// The value of str2 must be equal to the one of str1,
// because str1 does not contain characters to be escaped.
// However, str2 is diffrent from str1.
System.out.println(URLEncoder.encode(str1, "UTF-16BE")); //%D8%42%DF%B7A
System.out.println(URLEncoder.encode(str2, "UTF-16BE")); //%D8%42%DF%B7%FF%FD
The cause of this problem is that the loop to translate input character by character is wrong.
In CharSequenceTranslator.translate(CharSequence input, Writer out),
loop counter "i" moves from 0 to Character.codePointCount(input, 0, input.length()),
but it should move from 0 to input.length().


public int translate(CharSequence input, int index, Writer out) throws IOException {
    int seqEnd = input.length();
    // Uses -2 to ensure there is something after the &#
    if(input.charAt(index) == '&' && index < seqEnd - 1 && input.charAt(index + 1) == '#') {
        int start = index + 2;
        boolean isHex = false;

        char firstChar = input.charAt(start);
        if(firstChar == 'x' || firstChar == 'X') {
            start++;
            isHex = true;

            // Check there's more than just an x after the &#
        }

        int end = start;
        // Note that this supports character codes without a ; on the end
        while(input.charAt(end) != ';')
        {
            end++;
        }

        int entityValue;
        try {
            if(isHex) {
                entityValue = Integer.parseInt(input.subSequence(start, end).toString(), 16);
            } else {
                entityValue = Integer.parseInt(input.subSequence(start, end).toString(), 10);
            }
        } catch(NumberFormatException nfe) {
        System.err.println("FAIL: " + input.subSequence(start, end) + "[" + start +"]["+ end +"]");
            return 0;
        }

        if(entityValue > 0xFFFF) {
            char[] chrs = Character.toChars(entityValue);
            out.write(chrs[0]);
            out.write(chrs[1]);
        } else {
            out.write(entityValue);
        }

        return 2 + (end - start) + (isHex ? 1 : 0) + 1;
    }
    return 0;
}

public int translate(CharSequence input, int index, Writer out) throws IOException {
    int seqEnd = input.length();
    // Uses -2 to ensure there is something after the &#
    if(input.charAt(index) == '&' && index < seqEnd - 2 && input.charAt(index + 1) == '#') {
        int start = index + 2;
        boolean isHex = false;

        char firstChar = input.charAt(start);
        if(firstChar == 'x' || firstChar == 'X') {
            start++;
            isHex = true;

            // Check there's more than just an x after the &#
            if(start == seqEnd) {
                return 0;
            }
        }

        int end = start;
        // Note that this supports character codes without a ; on the end
        while(end < seqEnd && ( (input.charAt(end) >= '0' && input.charAt(end) <= '9') ||
                                (input.charAt(end) >= 'a' && input.charAt(end) <= 'f') ||
                                (input.charAt(end) >= 'A' && input.charAt(end) <= 'F') ) )
        {
            end++;
        }

        int entityValue;
        try {
            if(isHex) {
                entityValue = Integer.parseInt(input.subSequence(start, end).toString(), 16);
            } else {
                entityValue = Integer.parseInt(input.subSequence(start, end).toString(), 10);
            }
        } catch(NumberFormatException nfe) {
        System.err.println("FAIL: " + input.subSequence(start, end) + "[" + start +"]["+ end +"]");
            return 0;
        }

        if(entityValue > 0xFFFF) {
            char[] chrs = Character.toChars(entityValue);
            out.write(chrs[0]);
            out.write(chrs[1]);
        } else {
            out.write(entityValue);
        }

        boolean semiNext = (end != seqEnd) && (input.charAt(end) == ';');

        return 2 + (end - start) + (isHex ? 1 : 0) + (semiNext ? 1 : 0);
    }
    return 0;
}

StringIndexOutOfBoundsException when calling unescapeHtml4("&#03")

When calling unescapeHtml4() on the String "&#03" (or any String that contains these characters) an Exception is thrown:
Exception in thread "main" java.lang.StringIndexOutOfBoundsException: String index out of range: 4
	at java.lang.String.charAt(String.java:686)
	at org.apache.commons.lang3.text.translate.NumericEntityUnescaper.translate(NumericEntityUnescaper.java:49)
	at org.apache.commons.lang3.text.translate.AggregateTranslator.translate(AggregateTranslator.java:53)
	at org.apache.commons.lang3.text.translate.CharSequenceTranslator.translate(CharSequenceTranslator.java:88)
	at org.apache.commons.lang3.text.translate.CharSequenceTranslator.translate(CharSequenceTranslator.java:60)
	at org.apache.commons.lang3.StringEscapeUtils.unescapeHtml4(StringEscapeUtils.java:351)


public static boolean isSameLocalTime(Calendar cal1, Calendar cal2) {
    if (cal1 == null || cal2 == null) {
        throw new IllegalArgumentException("The date must not be null");
    }
    return (cal1.get(Calendar.MILLISECOND) == cal2.get(Calendar.MILLISECOND) &&
            cal1.get(Calendar.SECOND) == cal2.get(Calendar.SECOND) &&
            cal1.get(Calendar.MINUTE) == cal2.get(Calendar.MINUTE) &&
            cal1.get(Calendar.HOUR) == cal2.get(Calendar.HOUR) &&
            cal1.get(Calendar.DAY_OF_YEAR) == cal2.get(Calendar.DAY_OF_YEAR) &&
            cal1.get(Calendar.YEAR) == cal2.get(Calendar.YEAR) &&
            cal1.get(Calendar.ERA) == cal2.get(Calendar.ERA) &&
            cal1.getClass() == cal2.getClass());
}

public static boolean isSameLocalTime(Calendar cal1, Calendar cal2) {
    if (cal1 == null || cal2 == null) {
        throw new IllegalArgumentException("The date must not be null");
    }
    return (cal1.get(Calendar.MILLISECOND) == cal2.get(Calendar.MILLISECOND) &&
            cal1.get(Calendar.SECOND) == cal2.get(Calendar.SECOND) &&
            cal1.get(Calendar.MINUTE) == cal2.get(Calendar.MINUTE) &&
            cal1.get(Calendar.HOUR_OF_DAY) == cal2.get(Calendar.HOUR_OF_DAY) &&
            cal1.get(Calendar.DAY_OF_YEAR) == cal2.get(Calendar.DAY_OF_YEAR) &&
            cal1.get(Calendar.YEAR) == cal2.get(Calendar.YEAR) &&
            cal1.get(Calendar.ERA) == cal2.get(Calendar.ERA) &&
            cal1.getClass() == cal2.getClass());
}

DateUtils.isSameLocalTime does not work correct

Hi, I think I found a bug in the DateUtils class in the method isSameLocalTime.
Example:
Calendar a = Calendar.getInstance();
a.setTimeInMillis(1297364400000L);
Calendar b = Calendar.getInstance();
b.setTimeInMillis(1297321200000L);
Assert.assertFalse(DateUtils.isSameLocalTime(a, b));
This is because the method compares
cal1.get(Calendar.HOUR) == cal2.get(Calendar.HOUR)
but I think it has to be
cal1.get(Calendar.HOUR_OF_DAY) == cal2.get(Calendar.HOUR_OF_DAY)