protected Object _deserializeTypedUsingDefaultImpl(JsonParser p, DeserializationContext ctxt,
        TokenBuffer tb) throws IOException
{
    // As per [JACKSON-614], may have default implementation to use
    JsonDeserializer<Object> deser = _findDefaultImplDeserializer(ctxt);
    if (deser != null) {
        if (tb != null) {
            tb.writeEndObject();
            p = tb.asParser(p);
            // must move to point to the first token:
            p.nextToken();
        }
        return deser.deserialize(p, ctxt);
    }
    // or, perhaps we just bumped into a "natural" value (boolean/int/double/String)?
    Object result = TypeDeserializer.deserializeIfNatural(p, ctxt, _baseType);
    if (result != null) {
        return result;
    }
    // or, something for which "as-property" won't work, changed into "wrapper-array" type:
    if (p.getCurrentToken() == JsonToken.START_ARRAY) {
        return super.deserializeTypedFromAny(p, ctxt);
    }
    ctxt.reportWrongTokenException(p, JsonToken.FIELD_NAME,
            "missing property '"+_typePropertyName+"' that is to contain type id  (for class "+baseTypeName()+")");
    return null;
}

protected Object _deserializeTypedUsingDefaultImpl(JsonParser p, DeserializationContext ctxt,
        TokenBuffer tb) throws IOException
{
    // As per [JACKSON-614], may have default implementation to use
    JsonDeserializer<Object> deser = _findDefaultImplDeserializer(ctxt);
    if (deser != null) {
        if (tb != null) {
            tb.writeEndObject();
            p = tb.asParser(p);
            // must move to point to the first token:
            p.nextToken();
        }
        return deser.deserialize(p, ctxt);
    }
    // or, perhaps we just bumped into a "natural" value (boolean/int/double/String)?
    Object result = TypeDeserializer.deserializeIfNatural(p, ctxt, _baseType);
    if (result != null) {
        return result;
    }
    // or, something for which "as-property" won't work, changed into "wrapper-array" type:
    if (p.getCurrentToken() == JsonToken.START_ARRAY) {
        return super.deserializeTypedFromAny(p, ctxt);
    } else if (p.getCurrentToken() == JsonToken.VALUE_STRING) {
        if (ctxt.isEnabled(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT)) {
            String str = p.getText().trim();
            if (str.isEmpty()) {
                return null;
            }
        }
    }
    ctxt.reportWrongTokenException(p, JsonToken.FIELD_NAME,
            "missing property '"+_typePropertyName+"' that is to contain type id  (for class "+baseTypeName()+")");
    return null;
}

The AsPropertyTypeDeserializer  implementation does not respect the DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT feature. When deserializing an empty String it throws DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT instead of creating a null Object.


@Override
    public JsonDeserializer<Object> createBeanDeserializer(DeserializationContext ctxt,
            JavaType type, BeanDescription beanDesc)
        throws JsonMappingException
    {
        final DeserializationConfig config = ctxt.getConfig();
        // We may also have custom overrides:
        JsonDeserializer<Object> custom = _findCustomBeanDeserializer(type, config, beanDesc);
        if (custom != null) {
            return custom;
        }
        /* One more thing to check: do we have an exception type
         * (Throwable or its sub-classes)? If so, need slightly
         * different handling.
         */
        if (type.isThrowable()) {
            return buildThrowableDeserializer(ctxt, type, beanDesc);
        }
        /* Or, for abstract types, may have alternate means for resolution
         * (defaulting, materialization)
         */
        // 29-Nov-2015, tatu: Also, filter out calls to primitive types, they are
        //    not something we could materialize anything for
        if (type.isAbstract() && !type.isPrimitive()) {
            // Let's make it possible to materialize abstract types.
            JavaType concreteType = materializeAbstractType(ctxt, type, beanDesc);
            if (concreteType != null) {
                /* important: introspect actual implementation (abstract class or
                 * interface doesn't have constructors, for one)
                 */
                beanDesc = config.introspect(concreteType);
                return buildBeanDeserializer(ctxt, concreteType, beanDesc);
            }
        }

        // Otherwise, may want to check handlers for standard types, from superclass:
        @SuppressWarnings("unchecked")
        JsonDeserializer<Object> deser = (JsonDeserializer<Object>) findStdDeserializer(ctxt, type, beanDesc);
        if (deser != null) {
            return deser;
        }

        // Otherwise: could the class be a Bean class? If not, bail out
        if (!isPotentialBeanType(type.getRawClass())) {
            return null;
        }
        // For checks like [databind#1599]
        // Use generic bean introspection to build deserializer
        return buildBeanDeserializer(ctxt, type, beanDesc);
    }

@Override
    public JsonDeserializer<Object> createBeanDeserializer(DeserializationContext ctxt,
            JavaType type, BeanDescription beanDesc)
        throws JsonMappingException
    {
        final DeserializationConfig config = ctxt.getConfig();
        // We may also have custom overrides:
        JsonDeserializer<Object> custom = _findCustomBeanDeserializer(type, config, beanDesc);
        if (custom != null) {
            return custom;
        }
        /* One more thing to check: do we have an exception type
         * (Throwable or its sub-classes)? If so, need slightly
         * different handling.
         */
        if (type.isThrowable()) {
            return buildThrowableDeserializer(ctxt, type, beanDesc);
        }
        /* Or, for abstract types, may have alternate means for resolution
         * (defaulting, materialization)
         */
        // 29-Nov-2015, tatu: Also, filter out calls to primitive types, they are
        //    not something we could materialize anything for
        if (type.isAbstract() && !type.isPrimitive()) {
            // Let's make it possible to materialize abstract types.
            JavaType concreteType = materializeAbstractType(ctxt, type, beanDesc);
            if (concreteType != null) {
                /* important: introspect actual implementation (abstract class or
                 * interface doesn't have constructors, for one)
                 */
                beanDesc = config.introspect(concreteType);
                return buildBeanDeserializer(ctxt, concreteType, beanDesc);
            }
        }

        // Otherwise, may want to check handlers for standard types, from superclass:
        @SuppressWarnings("unchecked")
        JsonDeserializer<Object> deser = (JsonDeserializer<Object>) findStdDeserializer(ctxt, type, beanDesc);
        if (deser != null) {
            return deser;
        }

        // Otherwise: could the class be a Bean class? If not, bail out
        if (!isPotentialBeanType(type.getRawClass())) {
            return null;
        }
        // For checks like [databind#1599]
        checkIllegalTypes(ctxt, type, beanDesc);
        // Use generic bean introspection to build deserializer
        return buildBeanDeserializer(ctxt, type, beanDesc);
    }

Jackson Deserializer security vulnerability via default typing (CVE-2017-7525)


@Override
    public JsonDeserializer<Object> createBeanDeserializer(DeserializationContext ctxt,
            JavaType type, BeanDescription beanDesc)
        throws JsonMappingException
    {
        final DeserializationConfig config = ctxt.getConfig();
        // We may also have custom overrides:
        JsonDeserializer<Object> custom = _findCustomBeanDeserializer(type, config, beanDesc);
        if (custom != null) {
            return custom;
        }
        /* One more thing to check: do we have an exception type
         * (Throwable or its sub-classes)? If so, need slightly
         * different handling.
         */
        if (type.isThrowable()) {
            return buildThrowableDeserializer(ctxt, type, beanDesc);
        }
        /* Or, for abstract types, may have alternate means for resolution
         * (defaulting, materialization)
         */
        // 29-Nov-2015, tatu: Also, filter out calls to primitive types, they are
        //    not something we could materialize anything for
        if (type.isAbstract() && !type.isPrimitive() && !type.isEnumType()) {
            // Let's make it possible to materialize abstract types.
            JavaType concreteType = materializeAbstractType(ctxt, type, beanDesc);
            if (concreteType != null) {
                /* important: introspect actual implementation (abstract class or
                 * interface doesn't have constructors, for one)
                 */
                beanDesc = config.introspect(concreteType);
                return buildBeanDeserializer(ctxt, concreteType, beanDesc);
            }
        }
        // Otherwise, may want to check handlers for standard types, from superclass:
        @SuppressWarnings("unchecked")
        JsonDeserializer<Object> deser = (JsonDeserializer<Object>) findStdDeserializer(ctxt, type, beanDesc);
        if (deser != null) {
            return deser;
        }

        // Otherwise: could the class be a Bean class? If not, bail out
        if (!isPotentialBeanType(type.getRawClass())) {
            return null;
        }
        // For checks like [databind#1599]
        // Use generic bean introspection to build deserializer
        return buildBeanDeserializer(ctxt, type, beanDesc);
    }

@Override
    public JsonDeserializer<Object> createBeanDeserializer(DeserializationContext ctxt,
            JavaType type, BeanDescription beanDesc)
        throws JsonMappingException
    {
        final DeserializationConfig config = ctxt.getConfig();
        // We may also have custom overrides:
        JsonDeserializer<Object> custom = _findCustomBeanDeserializer(type, config, beanDesc);
        if (custom != null) {
            return custom;
        }
        /* One more thing to check: do we have an exception type
         * (Throwable or its sub-classes)? If so, need slightly
         * different handling.
         */
        if (type.isThrowable()) {
            return buildThrowableDeserializer(ctxt, type, beanDesc);
        }
        /* Or, for abstract types, may have alternate means for resolution
         * (defaulting, materialization)
         */
        // 29-Nov-2015, tatu: Also, filter out calls to primitive types, they are
        //    not something we could materialize anything for
        if (type.isAbstract() && !type.isPrimitive() && !type.isEnumType()) {
            // Let's make it possible to materialize abstract types.
            JavaType concreteType = materializeAbstractType(ctxt, type, beanDesc);
            if (concreteType != null) {
                /* important: introspect actual implementation (abstract class or
                 * interface doesn't have constructors, for one)
                 */
                beanDesc = config.introspect(concreteType);
                return buildBeanDeserializer(ctxt, concreteType, beanDesc);
            }
        }
        // Otherwise, may want to check handlers for standard types, from superclass:
        @SuppressWarnings("unchecked")
        JsonDeserializer<Object> deser = (JsonDeserializer<Object>) findStdDeserializer(ctxt, type, beanDesc);
        if (deser != null) {
            return deser;
        }

        // Otherwise: could the class be a Bean class? If not, bail out
        if (!isPotentialBeanType(type.getRawClass())) {
            return null;
        }
        // For checks like [databind#1599]
        checkIllegalTypes(ctxt, type, beanDesc);
        // Use generic bean introspection to build deserializer
        return buildBeanDeserializer(ctxt, type, beanDesc);
    }


protected void verifyNonDup(AnnotatedWithParams newOne, int typeIndex, boolean explicit)
{
    final int mask = (1 << typeIndex);
    _hasNonDefaultCreator = true;
    AnnotatedWithParams oldOne = _creators[typeIndex];
    // already had an explicitly marked one?
    if (oldOne != null) {

        if ((_explicitCreators & mask) != 0) { // already had explicitly annotated, leave as-is
            // but skip, if new one not annotated
            if (!explicit) {
                return;
            }
            // both explicit: verify
            // otherwise only verify if neither explicitly annotated.
        }

        // one more thing: ok to override in sub-class
        if (oldOne.getClass() == newOne.getClass()) {
            // [databind#667]: avoid one particular class of bogus problems

                throw new IllegalArgumentException("Conflicting "+TYPE_DESCS[typeIndex]
                        +" creators: already had explicitly marked "+oldOne+", encountered "+newOne);
            // otherwise, which one to choose?
                // new type more generic, use old
            // new type more specific, use it
        }
    }
    if (explicit) {
        _explicitCreators |= mask;
    }
    _creators[typeIndex] = _fixAccess(newOne);
}

protected void verifyNonDup(AnnotatedWithParams newOne, int typeIndex, boolean explicit)
{
    final int mask = (1 << typeIndex);
    _hasNonDefaultCreator = true;
    AnnotatedWithParams oldOne = _creators[typeIndex];
    // already had an explicitly marked one?
    if (oldOne != null) {
        boolean verify;

        if ((_explicitCreators & mask) != 0) { // already had explicitly annotated, leave as-is
            // but skip, if new one not annotated
            if (!explicit) {
                return;
            }
            // both explicit: verify
            verify = true;
        } else {
            // otherwise only verify if neither explicitly annotated.
            verify = !explicit;
        }

        // one more thing: ok to override in sub-class
        if (verify && (oldOne.getClass() == newOne.getClass())) {
            // [databind#667]: avoid one particular class of bogus problems
            Class<?> oldType = oldOne.getRawParameterType(0);
            Class<?> newType = newOne.getRawParameterType(0);

            if (oldType == newType) {
                throw new IllegalArgumentException("Conflicting "+TYPE_DESCS[typeIndex]
                        +" creators: already had explicitly marked "+oldOne+", encountered "+newOne);
            }
            // otherwise, which one to choose?
            if (newType.isAssignableFrom(oldType)) {
                // new type more generic, use old
                return;
            }
            // new type more specific, use it
        }
    }
    if (explicit) {
        _explicitCreators |= mask;
    }
    _creators[typeIndex] = _fixAccess(newOne);
}

Problem with bogus conflict between single-arg-String vs CharSequence constructor

Although it is good idea to allow recognizing CharSequence as almost like an alias for String, this can cause problems for classes like StringBuilder that have separate constructors for both.
This actually throws a bogus exception for 2.5.0, due to introduction of ability to recognize CharSequence.


    public JsonSerializer<?> createContextual(SerializerProvider serializers,
            BeanProperty property) throws JsonMappingException
    {
        if (property == null) {
            return this;
        }
        JsonFormat.Value format = findFormatOverrides(serializers, property, handledType());
        if (format == null) {
            return this;
        }
        // Simple case first: serialize as numeric timestamp?
        JsonFormat.Shape shape = format.getShape();
        if (shape.isNumeric()) {
            return withFormat(Boolean.TRUE, null);
        }

        // 08-Jun-2017, tatu: With [databind#1648], this gets bit tricky..
        // First: custom pattern will override things
                if ((shape == JsonFormat.Shape.STRING) || format.hasPattern()
                                || format.hasLocale() || format.hasTimeZone()) {
                    TimeZone tz = format.getTimeZone();
                    final String pattern = format.hasPattern()
                                    ? format.getPattern()
                                    : StdDateFormat.DATE_FORMAT_STR_ISO8601;
            final Locale loc = format.hasLocale()
                            ? format.getLocale()
                            : serializers.getLocale();
                    SimpleDateFormat df = new SimpleDateFormat(pattern, loc);
                    if (tz == null) {
                        tz = serializers.getTimeZone();
                    }
            df.setTimeZone(tz);
            return withFormat(Boolean.FALSE, df);
        }

        // Otherwise, need one of these changes:

        // Jackson's own `StdDateFormat` is quite easy to deal with...

        // 08-Jun-2017, tatu: Unfortunately there's no generally usable
        //    mechanism for changing `DateFormat` instances (or even clone()ing)
        //    So: require it be `SimpleDateFormat`; can't config other types
//            serializers.reportBadDefinition(handledType(), String.format(
            // Ugh. No way to change `Locale`, create copy; must re-crete completely:
        return this;
    }

    public JsonSerializer<?> createContextual(SerializerProvider serializers,
            BeanProperty property) throws JsonMappingException
    {
        if (property == null) {
            return this;
        }
        JsonFormat.Value format = findFormatOverrides(serializers, property, handledType());
        if (format == null) {
            return this;
        }
        // Simple case first: serialize as numeric timestamp?
        JsonFormat.Shape shape = format.getShape();
        if (shape.isNumeric()) {
            return withFormat(Boolean.TRUE, null);
        }

        // 08-Jun-2017, tatu: With [databind#1648], this gets bit tricky..
        // First: custom pattern will override things
        if (format.hasPattern()) {
            final Locale loc = format.hasLocale()
                            ? format.getLocale()
                            : serializers.getLocale();
            SimpleDateFormat df = new SimpleDateFormat(format.getPattern(), loc);
            TimeZone tz = format.hasTimeZone() ? format.getTimeZone()
                    : serializers.getTimeZone();
            df.setTimeZone(tz);
            return withFormat(Boolean.FALSE, df);
        }

        // Otherwise, need one of these changes:
        final boolean hasLocale = format.hasLocale();
        final boolean hasTZ = format.hasTimeZone();
        final boolean asString = (shape == JsonFormat.Shape.STRING);

        if (!hasLocale && !hasTZ && !asString) {
            return this;
        }

        DateFormat df0 = serializers.getConfig().getDateFormat();
        // Jackson's own `StdDateFormat` is quite easy to deal with...
        if (df0 instanceof StdDateFormat) {
            StdDateFormat std = (StdDateFormat) df0;
            if (format.hasLocale()) {
                std = std.withLocale(format.getLocale());
            }
            if (format.hasTimeZone()) {
                std = std.withTimeZone(format.getTimeZone());
            }
            return withFormat(Boolean.FALSE, std);
        }

        // 08-Jun-2017, tatu: Unfortunately there's no generally usable
        //    mechanism for changing `DateFormat` instances (or even clone()ing)
        //    So: require it be `SimpleDateFormat`; can't config other types
        if (!(df0 instanceof SimpleDateFormat)) {
//            serializers.reportBadDefinition(handledType(), String.format(
            serializers.reportMappingProblem(
"Configured `DateFormat` (%s) not a `SimpleDateFormat`; can not configure `Locale` or `TimeZone`",
df0.getClass().getName());
        }
        SimpleDateFormat df = (SimpleDateFormat) df0;
        if (hasLocale) {
            // Ugh. No way to change `Locale`, create copy; must re-crete completely:
            df = new SimpleDateFormat(df.toPattern(), format.getLocale());
        } else {
            df = (SimpleDateFormat) df.clone();
        }
        TimeZone newTz = format.getTimeZone();
        boolean changeTZ = (newTz != null) && !newTz.equals(df.getTimeZone());
        if (changeTZ) {
            df.setTimeZone(newTz);
        }
        return withFormat(Boolean.FALSE, df);
    }

DateTimeSerializerBase ignores configured date format when creating contextual

DateTimeSerializerBase#createContextual creates a new serializer with StdDateFormat.DATE_FORMAT_STR_ISO8601 format instead of re-using the actual format that may have been specified on the configuration. See the following code:
final String pattern = format.hasPattern()
                                    ? format.getPattern()
                                    : StdDateFormat.DATE_FORMAT_STR_ISO8601;

Using the @JsonFormat annotation on a field will therefore reset the format to Jackson's default even if the annotation doesn't specify any custom format.
DateBasedDeserializer#createContextual behaves differently and tries to re-use the configured format:
DateFormat df = ctxt.getConfig().getDateFormat();
// one shortcut: with our custom format, can simplify handling a bit
if (df.getClass() == StdDateFormat.class) {
   ...
   StdDateFormat std = (StdDateFormat) df;
   std = std.withTimeZone(tz);
   ...
} else {
  // otherwise need to clone, re-set timezone:
  df = (DateFormat) df.clone();
  df.setTimeZone(tz);
}

Shouldn't the serializer follow the same approach ?


protected JavaType _typeFromId(String id, DatabindContext ctxt) throws IOException
{
    /* 30-Jan-2010, tatu: Most ids are basic class names; so let's first
     *    check if any generics info is added; and only then ask factory
     *    to do translation when necessary
     */
    TypeFactory tf = ctxt.getTypeFactory();
    if (id.indexOf('<') > 0) {
        // note: may want to try combining with specialization (esp for EnumMap)?
        // 17-Aug-2017, tatu: As per [databind#1735] need to ensure assignment
        //    compatibility -- needed later anyway, and not doing so may open
        //    security issues.
        JavaType t = tf.constructFromCanonical(id);
            // Probably cleaner to have a method in `TypeFactory` but can't add in patch
        return t;
    }
    Class<?> cls;
    try {
        cls =  tf.findClass(id);
    } catch (ClassNotFoundException e) {
        // 24-May-2016, tatu: Ok, this is pretty ugly, but we should always get
        //   DeserializationContext, just playing it safe
        if (ctxt instanceof DeserializationContext) {
            DeserializationContext dctxt = (DeserializationContext) ctxt;
            // First: we may have problem handlers that can deal with it?
            return dctxt.handleUnknownTypeId(_baseType, id, this, "no such class found");
        }
        // ... meaning that we really should never get here.
        return null;
    } catch (Exception e) {
        throw new IllegalArgumentException("Invalid type id '"+id+"' (for id type 'Id.class'): "+e.getMessage(), e);
    }
    return tf.constructSpecializedType(_baseType, cls);
}

protected JavaType _typeFromId(String id, DatabindContext ctxt) throws IOException
{
    /* 30-Jan-2010, tatu: Most ids are basic class names; so let's first
     *    check if any generics info is added; and only then ask factory
     *    to do translation when necessary
     */
    TypeFactory tf = ctxt.getTypeFactory();
    if (id.indexOf('<') > 0) {
        // note: may want to try combining with specialization (esp for EnumMap)?
        // 17-Aug-2017, tatu: As per [databind#1735] need to ensure assignment
        //    compatibility -- needed later anyway, and not doing so may open
        //    security issues.
        JavaType t = tf.constructFromCanonical(id);
        if (!t.isTypeOrSubTypeOf(_baseType.getRawClass())) {
            // Probably cleaner to have a method in `TypeFactory` but can't add in patch
            throw new IllegalArgumentException(String.format(
                    "Class %s not subtype of %s", t.getRawClass().getName(), _baseType));
        }
        return t;
    }
    Class<?> cls;
    try {
        cls =  tf.findClass(id);
    } catch (ClassNotFoundException e) {
        // 24-May-2016, tatu: Ok, this is pretty ugly, but we should always get
        //   DeserializationContext, just playing it safe
        if (ctxt instanceof DeserializationContext) {
            DeserializationContext dctxt = (DeserializationContext) ctxt;
            // First: we may have problem handlers that can deal with it?
            return dctxt.handleUnknownTypeId(_baseType, id, this, "no such class found");
        }
        // ... meaning that we really should never get here.
        return null;
    } catch (Exception e) {
        throw new IllegalArgumentException("Invalid type id '"+id+"' (for id type 'Id.class'): "+e.getMessage(), e);
    }
    return tf.constructSpecializedType(_baseType, cls);
}

Missing type checks when using polymorphic type ids

(report by Lukes Euler)
JavaType supports limited amount of generic typing for textual representation, originally just to support typing needed for EnumMap (I think). Based on some reports, it appears that some of type compatibility checks are not performed in those cases; if so, they should be made since there is potential for abuse.
The problem here although actual type assignment will fail later on, ability to trigger some of processing (instantiation of incompatible classes, perhaps assingnment of properties) may itself be vulnerability.


public void validateSubType(DeserializationContext ctxt, JavaType type) throws JsonMappingException
    {
        // There are certain nasty classes that could cause problems, mostly
        // via default typing -- catch them here.
        final Class<?> raw = type.getRawClass();
        String full = raw.getName();

        main_check:
        do {
            if (_cfgIllegalClassNames.contains(full)) {
                break;
            }

            // 18-Dec-2017, tatu: As per [databind#1855], need bit more sophisticated handling
            //    for some Spring framework types
            // 05-Jan-2017, tatu: ... also, only applies to classes, not interfaces
            if (raw.isInterface()) {
                ;
            } else if (full.startsWith(PREFIX_SPRING)) {
                for (Class<?> cls = raw; (cls != null) && (cls != Object.class); cls = cls.getSuperclass()){
                    String name = cls.getSimpleName();
                    // looking for "AbstractBeanFactoryPointcutAdvisor" but no point to allow any is there?
                    if ("AbstractPointcutAdvisor".equals(name)
                            // ditto  for "FileSystemXmlApplicationContext": block all ApplicationContexts
                            || "AbstractApplicationContext".equals(name)) {
                        break main_check;
                    }
                // [databind#1737]; more 3rd party
                // s.add("com.mchange.v2.c3p0.JndiRefForwardingDataSource");
                // s.add("com.mchange.v2.c3p0.WrapperConnectionPoolDataSource");
                // [databind#1931]; more 3rd party
                // com.mchange.v2.c3p0.ComboPooledDataSource
                // com.mchange.v2.c3p0.debug.AfterCloseLoggingComboPooledDataSource
                }
            }
            return;
        } while (false);

        throw JsonMappingException.from(ctxt,
                String.format("Illegal type (%s) to deserialize: prevented for security reasons", full));
    }

public void validateSubType(DeserializationContext ctxt, JavaType type) throws JsonMappingException
    {
        // There are certain nasty classes that could cause problems, mostly
        // via default typing -- catch them here.
        final Class<?> raw = type.getRawClass();
        String full = raw.getName();

        main_check:
        do {
            if (_cfgIllegalClassNames.contains(full)) {
                break;
            }

            // 18-Dec-2017, tatu: As per [databind#1855], need bit more sophisticated handling
            //    for some Spring framework types
            // 05-Jan-2017, tatu: ... also, only applies to classes, not interfaces
            if (raw.isInterface()) {
                ;
            } else if (full.startsWith(PREFIX_SPRING)) {
                for (Class<?> cls = raw; (cls != null) && (cls != Object.class); cls = cls.getSuperclass()){
                    String name = cls.getSimpleName();
                    // looking for "AbstractBeanFactoryPointcutAdvisor" but no point to allow any is there?
                    if ("AbstractPointcutAdvisor".equals(name)
                            // ditto  for "FileSystemXmlApplicationContext": block all ApplicationContexts
                            || "AbstractApplicationContext".equals(name)) {
                        break main_check;
                    }
                }
            } else if (full.startsWith(PREFIX_C3P0)) {
                // [databind#1737]; more 3rd party
                // s.add("com.mchange.v2.c3p0.JndiRefForwardingDataSource");
                // s.add("com.mchange.v2.c3p0.WrapperConnectionPoolDataSource");
                // [databind#1931]; more 3rd party
                // com.mchange.v2.c3p0.ComboPooledDataSource
                // com.mchange.v2.c3p0.debug.AfterCloseLoggingComboPooledDataSource
                if (full.endsWith("DataSource")) {
                    break main_check;
                }
            }
            return;
        } while (false);

        throw JsonMappingException.from(ctxt,
                String.format("Illegal type (%s) to deserialize: prevented for security reasons", full));
    }

Block two more gadgets to exploit default typing issue (c3p0, CVE-2018-7489)

From an email report there are 2 other c3p0 classes (above and beyond ones listed in #1737) need to be blocked.

EDIT 21-Jun-2021: Fix included in:

* `2.9.5`
* `2.8.11.1`
* `2.7.9.3`
* `2.6.7.5`


protected void _addExplicitAnyCreator(DeserializationContext ctxt,
        BeanDescription beanDesc, CreatorCollector creators,
        CreatorCandidate candidate)
    throws JsonMappingException
{
    // Looks like there's bit of magic regarding 1-parameter creators; others simpler:
    if (1 != candidate.paramCount()) {
        // Ok: for delegates, we want one and exactly one parameter without
        // injection AND without name
        int oneNotInjected = candidate.findOnlyParamWithoutInjection();
        if (oneNotInjected >= 0) {
            // getting close; but most not have name
            if (candidate.paramName(oneNotInjected) == null) {
                _addExplicitDelegatingCreator(ctxt, beanDesc, creators, candidate);
                return;
            }
        }
        _addExplicitPropertyCreator(ctxt, beanDesc, creators, candidate);
        return;
    }
    AnnotatedParameter param = candidate.parameter(0);
    JacksonInject.Value injectId = candidate.injection(0);
    PropertyName paramName = candidate.explicitParamName(0);
    BeanPropertyDefinition paramDef = candidate.propertyDef(0);

    // If there's injection or explicit name, should be properties-based
    boolean useProps = (paramName != null) || (injectId != null);
    if (!useProps && (paramDef != null)) {
        // One more thing: if implicit name matches property with a getter
        // or field, we'll consider it property-based as well

        // 25-May-2018, tatu: as per [databind#2051], looks like we have to get
        //    not implicit name, but name with possible strategy-based-rename
          paramName = candidate.findImplicitParamName(0);
        paramName = candidate.findImplicitParamName(0);
        useProps = (paramName != null) && paramDef.couldSerialize();
    }
    if (useProps) {
        SettableBeanProperty[] properties = new SettableBeanProperty[] {
                constructCreatorProperty(ctxt, beanDesc, paramName, 0, param, injectId)
        };
        creators.addPropertyCreator(candidate.creator(), true, properties);
        return;
    }
    _handleSingleArgumentCreator(creators, candidate.creator(), true, true);

    // one more thing: sever link to creator property, to avoid possible later
    // problems with "unresolved" constructor property
    if (paramDef != null) {
        ((POJOPropertyBuilder) paramDef).removeConstructors();
    }
}

protected void _addExplicitAnyCreator(DeserializationContext ctxt,
        BeanDescription beanDesc, CreatorCollector creators,
        CreatorCandidate candidate)
    throws JsonMappingException
{
    // Looks like there's bit of magic regarding 1-parameter creators; others simpler:
    if (1 != candidate.paramCount()) {
        // Ok: for delegates, we want one and exactly one parameter without
        // injection AND without name
        int oneNotInjected = candidate.findOnlyParamWithoutInjection();
        if (oneNotInjected >= 0) {
            // getting close; but most not have name
            if (candidate.paramName(oneNotInjected) == null) {
                _addExplicitDelegatingCreator(ctxt, beanDesc, creators, candidate);
                return;
            }
        }
        _addExplicitPropertyCreator(ctxt, beanDesc, creators, candidate);
        return;
    }
    AnnotatedParameter param = candidate.parameter(0);
    JacksonInject.Value injectId = candidate.injection(0);
    PropertyName paramName = candidate.explicitParamName(0);
    BeanPropertyDefinition paramDef = candidate.propertyDef(0);

    // If there's injection or explicit name, should be properties-based
    boolean useProps = (paramName != null) || (injectId != null);
    if (!useProps && (paramDef != null)) {
        // One more thing: if implicit name matches property with a getter
        // or field, we'll consider it property-based as well

        // 25-May-2018, tatu: as per [databind#2051], looks like we have to get
        //    not implicit name, but name with possible strategy-based-rename
          paramName = candidate.findImplicitParamName(0);
        paramName = candidate.paramName(0);
        useProps = (paramName != null) && paramDef.couldSerialize();
    }
    if (useProps) {
        SettableBeanProperty[] properties = new SettableBeanProperty[] {
                constructCreatorProperty(ctxt, beanDesc, paramName, 0, param, injectId)
        };
        creators.addPropertyCreator(candidate.creator(), true, properties);
        return;
    }
    _handleSingleArgumentCreator(creators, candidate.creator(), true, true);

    // one more thing: sever link to creator property, to avoid possible later
    // problems with "unresolved" constructor property
    if (paramDef != null) {
        ((POJOPropertyBuilder) paramDef).removeConstructors();
    }
}

Implicit constructor property names are not renamed properly with PropertyNamingStrategy

(note: spin-off from FasterXML/jackson-modules-java8#67)
Looks like something with linking of creator properties (constructor arguments for annotated/discovered constructor) to "regular" properties does not work when using PropertyNamingStrategy. Apparently this was working better until 2.9.1, but broke with 2.9.2.


public final void serialize(JsonGenerator gen, SerializerProvider ctxt) throws IOException
{
    if (_value == null) {
        ctxt.defaultSerializeNull(gen);
    } else if (_value instanceof JsonSerializable) {
        ((JsonSerializable) _value).serialize(gen, ctxt);
    } else {
        // 25-May-2018, tatu: [databind#1991] do not call via generator but through context;
        //    this to preserve contextual information
        gen.writeObject(_value);
    }
}

public final void serialize(JsonGenerator gen, SerializerProvider ctxt) throws IOException
{
    if (_value == null) {
        ctxt.defaultSerializeNull(gen);
    } else if (_value instanceof JsonSerializable) {
        ((JsonSerializable) _value).serialize(gen, ctxt);
    } else {
        // 25-May-2018, tatu: [databind#1991] do not call via generator but through context;
        //    this to preserve contextual information
        ctxt.defaultSerializeValue(_value, gen);
    }
}

Context attributes are not passed/available to custom serializer if object is in POJO

Below is a test case where I create a custom serializer and use it to serialize an object 1) in a HashMap and 2) in an ObjectNode. In both cases I pass attribute to the serializer like this:
mapper.writer().withAttribute("myAttr", "Hello!")
Serializing HashMap works as expected, but during ObjectNode serialization the attribute is null . It seems that in both cases the custom serializer should get access to the passed attribute and so both lines in the output should contain "Hello!"
Produced output from running testCase.test()
{"data":{"aStr":"The value is: Hello!"}}
{"data":{"aStr":"The value is: NULL"}}

Test case:
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

public class TestCase {
  public final static ObjectMapper mapper = new ObjectMapper();

  @JsonSerialize(using = TestCase.CustomSer.class)
  public static class Data {
    public String aStr;
  }

  public static class CustomSer extends StdSerializer<Data> {
    public CustomSer() {
      super(Data.class);
    }

    @Override
    public void serialize(Data value, JsonGenerator gen, SerializerProvider provider) throws IOException {
      String attrStr = (String) provider.getAttribute("myAttr");
      gen.writeStartObject();
      gen.writeObjectField("aStr", "The value is: " + (attrStr == null ? "NULL" : attrStr));
      gen.writeEndObject();
    }
  }

  public static void test() throws IOException {
    Data data = new Data();
    data.aStr = "Hello";

    Map<String, Object> mapTest = new HashMap<>();
    mapTest.put("data", data);

    ObjectNode treeTest = mapper.createObjectNode();
    treeTest.putPOJO("data", data);

    String mapOut = mapper.writer().withAttribute("myAttr", "Hello!").writeValueAsString(mapTest);
    System.out.println(mapOut);

    String treeOut = mapper.writer().withAttribute("myAttr", "Hello!").writeValueAsString(treeTest);
    System.out.println(treeOut);
  }
}


protected String buildCanonicalName()
{
    StringBuilder sb = new StringBuilder();
    sb.append(_class.getName());
    sb.append('<');
    sb.append(_referencedType.toCanonical());
    return sb.toString();
}

protected String buildCanonicalName()
{
    StringBuilder sb = new StringBuilder();
    sb.append(_class.getName());
    sb.append('<');
    sb.append(_referencedType.toCanonical());
    sb.append('>');
    return sb.toString();
}

Canonical string for reference type is built incorrectly.
E.g.:
new ReferenceType(new TypeFactory(new LRUMap<Object, JavaType>(0, 10000)).constructType(Object.class), new PlaceholderForType(0)).toCanonical()
yields:
java.lang.Object<$1
while the expected value is:
java.lang.Object<$1>


private final int _next() throws XMLStreamException
    {
        switch (_currentState) {
        case XML_ATTRIBUTE_VALUE:
            ++_nextAttributeIndex;
            // fall through
        case XML_START_ELEMENT: // attributes to return?
            if (_nextAttributeIndex < _attributeCount) {
                _localName = _xmlReader.getAttributeLocalName(_nextAttributeIndex);
                _namespaceURI = _xmlReader.getAttributeNamespace(_nextAttributeIndex);
                _textValue = _xmlReader.getAttributeValue(_nextAttributeIndex);
                return (_currentState = XML_ATTRIBUTE_NAME);
            }
            // otherwise need to find START/END_ELEMENT or text
            String text = _collectUntilTag();
            // If we have no/all-whitespace text followed by START_ELEMENT, ignore text
            if (_xmlReader.getEventType() == XMLStreamReader.START_ELEMENT) {
                    return _initStartElement();
            }
            // For END_ELEMENT we will return text, if any
            if (text != null) {
                _textValue = text;
                return (_currentState = XML_TEXT);
            }
            return _handleEndElement();

        case XML_ATTRIBUTE_NAME:
            // if we just returned name, will need to just send value next
            return (_currentState = XML_ATTRIBUTE_VALUE);
        case XML_TEXT:
            // mixed text with other elements
            // text followed by END_ELEMENT
            return _handleEndElement();
        case XML_END:
            return XML_END;
//            throw new IllegalStateException("No more XML tokens available (end of input)");
        }

        // Ok: must be END_ELEMENT; see what tag we get (or end)
        switch (_skipUntilTag()) {
        case XMLStreamConstants.END_DOCUMENT:
            return (_currentState = XML_END);
        case XMLStreamConstants.END_ELEMENT:
            return _handleEndElement();
        }
        // START_ELEMENT...
        return _initStartElement();
    }

private final int _next() throws XMLStreamException
    {
        switch (_currentState) {
        case XML_ATTRIBUTE_VALUE:
            ++_nextAttributeIndex;
            // fall through
        case XML_START_ELEMENT: // attributes to return?
            if (_nextAttributeIndex < _attributeCount) {
                _localName = _xmlReader.getAttributeLocalName(_nextAttributeIndex);
                _namespaceURI = _xmlReader.getAttributeNamespace(_nextAttributeIndex);
                _textValue = _xmlReader.getAttributeValue(_nextAttributeIndex);
                return (_currentState = XML_ATTRIBUTE_NAME);
            }
            // otherwise need to find START/END_ELEMENT or text
            String text = _collectUntilTag();
            final boolean startElementNext = _xmlReader.getEventType() == XMLStreamReader.START_ELEMENT;
            // If we have no/all-whitespace text followed by START_ELEMENT, ignore text
            if (startElementNext) {
                if (text == null || _allWs(text)) {
                    _mixedText = false;
                    return _initStartElement();
                }
                _mixedText = true;
                _textValue = text;
                return (_currentState = XML_TEXT);
            }
            // For END_ELEMENT we will return text, if any
            if (text != null) {
                _mixedText = false;
                _textValue = text;
                return (_currentState = XML_TEXT);
            }
            _mixedText = false;
            return _handleEndElement();

        case XML_ATTRIBUTE_NAME:
            // if we just returned name, will need to just send value next
            return (_currentState = XML_ATTRIBUTE_VALUE);
        case XML_TEXT:
            // mixed text with other elements
            if (_mixedText){
                _mixedText = false;
                return _initStartElement();
            }
            // text followed by END_ELEMENT
            return _handleEndElement();
        case XML_END:
            return XML_END;
//            throw new IllegalStateException("No more XML tokens available (end of input)");
        }

        // Ok: must be END_ELEMENT; see what tag we get (or end)
        switch (_skipUntilTag()) {
        case XMLStreamConstants.END_DOCUMENT:
            return (_currentState = XML_END);
        case XMLStreamConstants.END_ELEMENT:
            return _handleEndElement();
        }
        // START_ELEMENT...
        return _initStartElement();
    }

Mixed content not supported if there are child elements.

@XmlText is only supported if there are no child elements, support could be improved with some changes in XmlTokenStream.
I successfully made some changes in XmlTokenStream, it's working in my personal case, but it needs more tests.
If agreed, I could provide a patch.

Example:
Input string : `"<windSpeed units=\"kt\">27<radius>20</radius></windSpeed>"`
 "CxmlWindSpeed" class :

```
public class WindSpeed {

    public static class Radius {
        @JacksonXmlProperty(isAttribute = true)
        private String sector;
        @JacksonXmlProperty(isAttribute = true)
        private String units;
        @JacksonXmlText
        private int value;
        ..../ Getters and Setters code/....
    }
    @JacksonXmlProperty(isAttribute = true)
    private String units;
    @JacksonXmlProperty(isAttribute = true)
    private String source;
    @JacksonXmlText
    private int value;
    @JacksonXmlElementWrapper(useWrapping = false)
    private List<Radius> radius;
    ..../ Getters and Setters code/....
}
```


protected void _serializeXmlNull(JsonGenerator jgen) throws IOException
{
    // 14-Nov-2016, tatu: As per [dataformat-xml#213], we may have explicitly
    //    configured root name...
    if (jgen instanceof ToXmlGenerator) {
        _initWithRootName((ToXmlGenerator) jgen, ROOT_NAME_FOR_NULL);
    }
    super.serializeValue(jgen, null);
}

protected void _serializeXmlNull(JsonGenerator jgen) throws IOException
{
    // 14-Nov-2016, tatu: As per [dataformat-xml#213], we may have explicitly
    //    configured root name...
    QName rootName = _rootNameFromConfig();
    if (rootName == null) {
        rootName = ROOT_NAME_FOR_NULL;
    }
    if (jgen instanceof ToXmlGenerator) {
        _initWithRootName((ToXmlGenerator) jgen, rootName);
    }
    super.serializeValue(jgen, null);
}

XmlSerializerProvider does not use withRootName config for null

In jackson-dataformat-xml/src/main/java/com/fasterxml/jackson/dataformat/xml/ser/XmlSerializerProvider.java
Line 203, I think _rootNameFromConfig() should be used if available instead of ROOT_NAME_FOR_NULL, so that withRootName() config can be used.
I don't know whether/how deser would be affected

jackson-dataformat-xml/src/main/java/com/fasterxml/jackson/dataformat/xml/ser/XmlSerializerProvider.java

         Line 203
      in
      ca1c671

 _initWithRootName((ToXmlGenerator) jgen, ROOT_NAME_FOR_NULL);


protected XmlSerializerProvider(XmlSerializerProvider src) {
    super(src);
    // 21-May-2018, tatu: As per [dataformat-xml#282], should NOT really copy
    //    root name lookup as that may link back to diff version, configuration
    _rootNameLookup = src._rootNameLookup;
}

protected XmlSerializerProvider(XmlSerializerProvider src) {
    super(src);
    // 21-May-2018, tatu: As per [dataformat-xml#282], should NOT really copy
    //    root name lookup as that may link back to diff version, configuration
    _rootNameLookup = new XmlRootNameLookup();
}

@JacksonXmlRootElement malfunction when using it with multiple XmlMappers and disabling annotations

Found this in version 2.9.4 running some tests that go back and forth serializing with an XML mapper that uses annotations, and another one that ignores them. May be related to issue #171 and the cache of class annotations.
When running this code, the second print statement should use the annotation's localName but it instead uses the class name.
@JacksonXmlRootElement(localName = "myname")
public class XMLTest {

    public static void main(String[] s) throws Exception {

        final ObjectMapper xmlMapper = new XmlMapper();
        final ObjectMapper noAnnotationsXmlMapper = xmlMapper.copy()
                .configure(MapperFeature.USE_ANNOTATIONS, false)
                .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);

        System.out.println(noAnnotationsXmlMapper.writeValueAsString(new XMLTest()));
        System.out.println(xmlMapper.writeValueAsString(new XMLTest()));

    }

}

Output:
<XMLTest/>
<XMLTest/>


private void normalise(Element element) {
    List<Node> toMove = new ArrayList<Node>();
    for (Node node: element.childNodes) {
        if (node instanceof TextNode) {
            TextNode tn = (TextNode) node;
            if (!tn.isBlank())
                toMove.add(tn);
        }
    }

    for (Node node: toMove) {
        element.removeChild(node);
        body().appendChild(new TextNode(" ", ""));
        body().appendChild(node);
    }
}

private void normalise(Element element) {
    List<Node> toMove = new ArrayList<Node>();
    for (Node node: element.childNodes) {
        if (node instanceof TextNode) {
            TextNode tn = (TextNode) node;
            if (!tn.isBlank())
                toMove.add(tn);
        }
    }

    for (Node node: toMove) {
        element.removeChild(node);
        body().prependChild(node);
        body().prependChild(new TextNode(" ", ""));
    }
}

Parsing a HTML snippet causes the leading text to be moved to back

Code:
String html = "foo <b>bar</b> baz";
String text = Jsoup.parse(html).text();
System.out.println(text);

Result:
bar baz foo

Expected:
foo bar baz


public String absUrl(String attributeKey) {
    Validate.notEmpty(attributeKey);

    String relUrl = attr(attributeKey);
    if (!hasAttr(attributeKey)) {
        return ""; // nothing to make absolute with
    } else {
        URL base;
        try {
            try {
                base = new URL(baseUri);
            } catch (MalformedURLException e) {
                // the base is unsuitable, but the attribute may be abs on its own, so try that
                URL abs = new URL(relUrl);
                return abs.toExternalForm();
            }
            // workaround: java resolves '//path/file + ?foo' to '//path/?foo', not '//path/file?foo' as desired
            URL abs = new URL(base, relUrl);
            return abs.toExternalForm();
        } catch (MalformedURLException e) {
            return "";
        }
    }
}

public String absUrl(String attributeKey) {
    Validate.notEmpty(attributeKey);

    String relUrl = attr(attributeKey);
    if (!hasAttr(attributeKey)) {
        return ""; // nothing to make absolute with
    } else {
        URL base;
        try {
            try {
                base = new URL(baseUri);
            } catch (MalformedURLException e) {
                // the base is unsuitable, but the attribute may be abs on its own, so try that
                URL abs = new URL(relUrl);
                return abs.toExternalForm();
            }
            // workaround: java resolves '//path/file + ?foo' to '//path/?foo', not '//path/file?foo' as desired
            if (relUrl.startsWith("?"))
                relUrl = base.getPath() + relUrl;
            URL abs = new URL(base, relUrl);
            return abs.toExternalForm();
        } catch (MalformedURLException e) {
            return "";
        }
    }
}

Document doc = Jsoup.parse(new URL("http://www.oschina.net/bbs/thread/12975"), 5*1000);
Elements es = doc.select("a[href]");
for(Iterator it = es.iterator();it.hasNext();){
Element e = it.next();
System.out.println(e.absUrl("href"));
}
attr("abs:href")   ------  1
result: -------------------   http://www.oschina.net/bbs/thread/?p=1
I think it's a wrong result~.
The correct results should be "http://www.oschina.net/bbs/thread/12975?p=1"


public boolean hasAttr(String attributeKey) {
    Validate.notNull(attributeKey);

    return attributes.hasKey(attributeKey);
}

public boolean hasAttr(String attributeKey) {
    Validate.notNull(attributeKey);

    if (attributeKey.toLowerCase().startsWith("abs:")) {
        String key = attributeKey.substring("abs:".length());
        if (attributes.hasKey(key) && !absUrl(key).equals(""))
            return true;
    }
    return attributes.hasKey(attributeKey);
}

abs: attribute prefix does not work on Elements.attr()

Elements.attr() iterates on its element to look for the first one with the given attrbute.
If I try to get the attribute abs:href, the test element.hasAttr("abs:herf") fails, and the returned value is an empty string.


private boolean testValidProtocol(Element el, Attribute attr, Set<Protocol> protocols) {
    // try to resolve relative urls to abs, and optionally update the attribute so output html has abs.
    // rels without a baseuri get removed
    String value = el.absUrl(attr.getKey());
    if (!preserveRelativeLinks)
        attr.setValue(value);

    for (Protocol protocol : protocols) {
        String prot = protocol.toString() + ":";
        if (value.toLowerCase().startsWith(prot)) {
            return true;
        }
    }
    return false;
}

private boolean testValidProtocol(Element el, Attribute attr, Set<Protocol> protocols) {
    // try to resolve relative urls to abs, and optionally update the attribute so output html has abs.
    // rels without a baseuri get removed
    String value = el.absUrl(attr.getKey());
    if (value.length() == 0)
        value = attr.getValue(); // if it could not be made abs, run as-is to allow custom unknown protocols
    if (!preserveRelativeLinks)
        attr.setValue(value);

    for (Protocol protocol : protocols) {
        String prot = protocol.toString() + ":";
        if (value.toLowerCase().startsWith(prot)) {
            return true;
        }
    }
    return false;
}

Cleaning html containing the cid identifier breaks images

Ok, so in mail type HTML the following is common

The item after CID: can be almost anything (US-ASCII I think) and of any length. It corresponds to an image linked elsewhere in MIME say like this
--mimebounday
Content-ID:
Content-Type: image/jpeg.....
(snip)
So, to mark a long story somewhat shorter, I use Jsoup's sanitizer extensively. However, I need these CID references to be preserved post sanitization. addProtocols does not work because the items are not valid URLs. As a result
the above becomes . Which for my purposes is not good :)


private void parseStartTag() {
    tq.consume("<");
    String tagName = tq.consumeWord();

    if (tagName.length() == 0) { // doesn't look like a start tag after all; put < back on stack and handle as text
        tq.addFirst("<");
        parseTextNode();
        return;
    }

    Attributes attributes = new Attributes();
    while (!tq.matchesAny("<", "/>", ">") && !tq.isEmpty()) {
        Attribute attribute = parseAttribute();
        if (attribute != null)
            attributes.put(attribute);
    }

    Tag tag = Tag.valueOf(tagName);
    Element child = new Element(tag, baseUri, attributes);

    boolean isEmptyElement = tag.isEmpty(); // empty element if empty tag (e.g. img) or self-closed el (<div/>
    if (tq.matchChomp("/>")) { // close empty element or tag
        isEmptyElement = true;
    } else {
        tq.matchChomp(">");
    }
    addChildToParent(child, isEmptyElement);

    // pc data only tags (textarea, script): chomp to end tag, add content as text node
    if (tag.isData()) {
        String data = tq.chompTo("</" + tagName);
        tq.chompTo(">");

        Node dataNode;
        if (tag.equals(titleTag) || tag.equals(textareaTag)) // want to show as text, but not contain inside tags (so not a data tag?)
            dataNode = TextNode.createFromEncoded(data, baseUri);
        else
            dataNode = new DataNode(data, baseUri); // data not encoded but raw (for " in script)
        child.appendChild(dataNode);
    }

    // <base href>: update the base uri
    if (child.tagName().equals("base")) {
        String href = child.absUrl("href");
        if (href.length() != 0) { // ignore <base target> etc
            baseUri = href;
            doc.setBaseUri(href); // set on the doc so doc.createElement(Tag) will get updated base
        }
    }
}

private void parseStartTag() {
    tq.consume("<");
    String tagName = tq.consumeWord();

    if (tagName.length() == 0) { // doesn't look like a start tag after all; put < back on stack and handle as text
        tq.addFirst("<");
        parseTextNode();
        return;
    }

    Attributes attributes = new Attributes();
    while (!tq.matchesAny("<", "/>", ">") && !tq.isEmpty()) {
        Attribute attribute = parseAttribute();
        if (attribute != null)
            attributes.put(attribute);
    }

    Tag tag = Tag.valueOf(tagName);
    Element child = new Element(tag, baseUri, attributes);

    boolean isEmptyElement = tag.isEmpty(); // empty element if empty tag (e.g. img) or self-closed el (<div/>
    if (tq.matchChomp("/>")) { // close empty element or tag
        isEmptyElement = true;
    } else {
        tq.matchChomp(">");
    }
    addChildToParent(child, isEmptyElement);

    // pc data only tags (textarea, script): chomp to end tag, add content as text node
    if (tag.isData()) {
        String data = tq.chompTo("</" + tagName);
        tq.chompTo(">");
        popStackToClose(tag);

        Node dataNode;
        if (tag.equals(titleTag) || tag.equals(textareaTag)) // want to show as text, but not contain inside tags (so not a data tag?)
            dataNode = TextNode.createFromEncoded(data, baseUri);
        else
            dataNode = new DataNode(data, baseUri); // data not encoded but raw (for " in script)
        child.appendChild(dataNode);
    }

    // <base href>: update the base uri
    if (child.tagName().equals("base")) {
        String href = child.absUrl("href");
        if (href.length() != 0) { // ignore <base target> etc
            baseUri = href;
            doc.setBaseUri(href); // set on the doc so doc.createElement(Tag) will get updated base
        }
    }
}

Unadorned text following data-only tags doesn't parse properly

This HTML, parsed and immediately printed out, results in:
<html>
<body>
<script type="text/javascript">
var inside = true;
</script>
this should be outside.
</body>
</html>
Results:
<html>
<head>
</head>
<body>
<script type="text/javascript">
var inside = true;
this should be outside.
</script>
</body>
</html>
Note how "this should be outside" ends up inside the <script> tag, instead of following it.  From what I can tell, this only happens to data-only tags.


static Document parseByteData(ByteBuffer byteData, String charsetName, String baseUri, Parser parser) {
    String docData;
    Document doc = null;
    if (charsetName == null) { // determine from meta. safe parse as UTF-8
        // look for <meta http-equiv="Content-Type" content="text/html;charset=gb2312"> or HTML5 <meta charset="gb2312">
        docData = Charset.forName(defaultCharset).decode(byteData).toString();
        doc = parser.parseInput(docData, baseUri);
        Element meta = doc.select("meta[http-equiv=content-type], meta[charset]").first();
        if (meta != null) { // if not found, will keep utf-8 as best attempt
            String foundCharset = meta.hasAttr("http-equiv") ? getCharsetFromContentType(meta.attr("content")) : meta.attr("charset");
            if (foundCharset != null && foundCharset.length() != 0 && !foundCharset.equals(defaultCharset)) { // need to re-decode
                charsetName = foundCharset;
                byteData.rewind();
                docData = Charset.forName(foundCharset).decode(byteData).toString();
                doc = null;
            }
        }
    } else { // specified by content type header (or by user on file load)
        Validate.notEmpty(charsetName, "Must set charset arg to character set of file to parse. Set to null to attempt to detect from HTML");
        docData = Charset.forName(charsetName).decode(byteData).toString();
    }
    if (doc == null) {
        // there are times where there is a spurious byte-order-mark at the start of the text. Shouldn't be present
        // in utf-8. If after decoding, there is a BOM, strip it; otherwise will cause the parser to go straight
        // into head mode

        doc = parser.parseInput(docData, baseUri);
        doc.outputSettings().charset(charsetName);
    }
    return doc;
}

static Document parseByteData(ByteBuffer byteData, String charsetName, String baseUri, Parser parser) {
    String docData;
    Document doc = null;
    if (charsetName == null) { // determine from meta. safe parse as UTF-8
        // look for <meta http-equiv="Content-Type" content="text/html;charset=gb2312"> or HTML5 <meta charset="gb2312">
        docData = Charset.forName(defaultCharset).decode(byteData).toString();
        doc = parser.parseInput(docData, baseUri);
        Element meta = doc.select("meta[http-equiv=content-type], meta[charset]").first();
        if (meta != null) { // if not found, will keep utf-8 as best attempt
            String foundCharset = meta.hasAttr("http-equiv") ? getCharsetFromContentType(meta.attr("content")) : meta.attr("charset");
            if (foundCharset != null && foundCharset.length() != 0 && !foundCharset.equals(defaultCharset)) { // need to re-decode
                charsetName = foundCharset;
                byteData.rewind();
                docData = Charset.forName(foundCharset).decode(byteData).toString();
                doc = null;
            }
        }
    } else { // specified by content type header (or by user on file load)
        Validate.notEmpty(charsetName, "Must set charset arg to character set of file to parse. Set to null to attempt to detect from HTML");
        docData = Charset.forName(charsetName).decode(byteData).toString();
    }
    if (doc == null) {
        // there are times where there is a spurious byte-order-mark at the start of the text. Shouldn't be present
        // in utf-8. If after decoding, there is a BOM, strip it; otherwise will cause the parser to go straight
        // into head mode
        if (docData.charAt(0) == 65279)
            docData = docData.substring(1);

        doc = parser.parseInput(docData, baseUri);
        doc.outputSettings().charset(charsetName);
    }
    return doc;
}

Some html file's head element will be empty

Hello, Jonathan
I love Jsoup, and handling many html files.
But today, I'm under the  problem.
When parse with Jsoup, some html file's head element will be empty.
Sample html is here -> http://dl.dropbox.com/u/972460/test.html
Please help me.


void read(Tokeniser t, CharacterReader r) {
    if (r.matchesLetter()) {
        String name = r.consumeLetterSequence();
        t.tagPending.appendTagName(name.toLowerCase());
        t.dataBuffer.append(name);
        r.advance();
        return;
    }

    if (t.isAppropriateEndTagToken() && !r.isEmpty()) {
        char c = r.consume();
        switch (c) {
            case '\t':
            case '\n':
            case '\f':
            case ' ':
                t.transition(BeforeAttributeName);
                break;
            case '/':
                t.transition(SelfClosingStartTag);
                break;
            case '>':
                t.emitTagPending();
                t.transition(Data);
                break;
            default:
                t.dataBuffer.append(c);
                anythingElse(t, r);
                break;
        }
    } else {
        anythingElse(t, r);
    }
}

void read(Tokeniser t, CharacterReader r) {
    if (r.matchesLetter()) {
        String name = r.consumeLetterSequence();
        t.tagPending.appendTagName(name.toLowerCase());
        t.dataBuffer.append(name);
        return;
    }

    if (t.isAppropriateEndTagToken() && !r.isEmpty()) {
        char c = r.consume();
        switch (c) {
            case '\t':
            case '\n':
            case '\f':
            case ' ':
                t.transition(BeforeAttributeName);
                break;
            case '/':
                t.transition(SelfClosingStartTag);
                break;
            case '>':
                t.emitTagPending();
                t.transition(Data);
                break;
            default:
                t.dataBuffer.append(c);
                anythingElse(t, r);
                break;
        }
    } else {
        anythingElse(t, r);
    }
}

1.6.0 dropping a ' on a particular javascript string

Loses a single quote when the javascript contains a partial tag, exampled pared from ad section of http://scienceblogs.com/pharyngula.  Note in the result that '</scr is missing closing ' :
Input:
<HTML>
<body>
 <div>
  <script language="JavaScript1.1">
    document.write('</scr' + 'ipt>');
  </script>
 </div>
</body>
</HTML>

Result:
<html>
 <body>
  <div>
   <script language="JavaScript1.1">
    document.write('</scr + 'ipt>');

   </script>
  </div>
 </body>
</html>


public Document clean(Document dirtyDocument) {
    Validate.notNull(dirtyDocument);

    Document clean = Document.createShell(dirtyDocument.baseUri());
        copySafeNodes(dirtyDocument.body(), clean.body());

    return clean;
}

public Document clean(Document dirtyDocument) {
    Validate.notNull(dirtyDocument);

    Document clean = Document.createShell(dirtyDocument.baseUri());
    if (dirtyDocument.body() != null) // frameset documents won't have a body. the clean doc will have empty body.
        copySafeNodes(dirtyDocument.body(), clean.body());

    return clean;
}

NullpointerException when applying Cleaner to a frameset

To reproduce:

Create/find a html document of a frameset.
Parse the html.
Create a Cleaner instance and call the clean method with the document from step 2.
NullPointerException

Cause:
In Cleaner.clean(Document) (https://github.com/jhy/jsoup/blob/master/src/main/java/org/jsoup/safety/Cleaner.java#L43) the copySafeNodes is called with the document.body(). However, this is null when handling a frameset document.
Expected:
An empty document or perhaps null returned. But not a nullpointerException.


static String getCharsetFromContentType(String contentType) {
    if (contentType == null) return null;
    Matcher m = charsetPattern.matcher(contentType);
    if (m.find()) {
        String charset = m.group(1).trim();
        charset = charset.toUpperCase(Locale.ENGLISH);
        return charset;
    }
    return null;
}

static String getCharsetFromContentType(String contentType) {
    if (contentType == null) return null;
    Matcher m = charsetPattern.matcher(contentType);
    if (m.find()) {
        String charset = m.group(1).trim();
        if (Charset.isSupported(charset)) return charset;
        charset = charset.toUpperCase(Locale.ENGLISH);
        if (Charset.isSupported(charset)) return charset;
    }
    return null;
}

Invalid HTTP-Response header leads to exception

In particular case a HTTP-Webpage responses with a invalid HTTP-Charset field (delivered UFT8 instead of UTF8).
This leads to an UnsupportedCharsetException in org.jsoup.helper.DataUtil at around Line 93(?) where :
  Validate.notEmpty(charsetName, "Must set charset arg to character set of file to parse. Set to null to attempt to detect from HTML");
  docData = Charset.forName(charsetName).decode(byteData).toString();
I fixed it by wrapping a try catch statement around these two lines such that:
try{
  Validate.notEmpty(charsetName, "Must set charset arg to character set of file to parse. Set to null to attempt to detect from HTML");
  docData = Charset.forName(charsetName).decode(byteData).toString();
} catch(UnsupportedCharsetException e){
  return parseByteData(byteData,(String)null,baseUri,parser);
}
It now falls back to the none charset argument assigned clause, and tries to detect the character set via HTML.


public Element clone() {
    Element clone = (Element) super.clone();
    clone.classNames();
    return clone;
}

public Element clone() {
    Element clone = (Element) super.clone();
    clone.classNames = null; // derived on first hit, otherwise gets a pointer to source classnames
    return clone;
}

Element.clone() wrongly shared a same classNames Set instance

In the clone() method of Node, the Object.clone() is called, if the original element's classNames Set had been initialized before clone, the original classNames Set will be set to the new cloned Element instance due to the JDK's clone mechanism. Thus, the old element and the newly cloned Element will share a same classNames Set instance.


Element insert(Token.StartTag startTag) {
    // handle empty unknown tags
    // when the spec expects an empty tag, will directly hit insertEmpty, so won't generate this fake end tag.
    if (startTag.isSelfClosing()) {
        Element el = insertEmpty(startTag);
        stack.add(el);
        tokeniser.emit(new Token.EndTag(el.tagName()));  // ensure we get out of whatever state we are in. emitted for yielded processing
        return el;
    }

    Element el = new Element(Tag.valueOf(startTag.name()), baseUri, startTag.attributes);
    insert(el);
    return el;
}

Element insert(Token.StartTag startTag) {
    // handle empty unknown tags
    // when the spec expects an empty tag, will directly hit insertEmpty, so won't generate this fake end tag.
    if (startTag.isSelfClosing()) {
        Element el = insertEmpty(startTag);
        stack.add(el);
        tokeniser.transition(TokeniserState.Data); // handles <script />, otherwise needs breakout steps from script data
        tokeniser.emit(new Token.EndTag(el.tagName()));  // ensure we get out of whatever state we are in. emitted for yielded processing
        return el;
    }

    Element el = new Element(Tag.valueOf(startTag.name()), baseUri, startTag.attributes);
    insert(el);
    return el;
}

Self-closing script tag causes remainder of document to be html-escaped.

When a self-closing script block is encountered it appears that the state transitions do not account for the closing tag, so the rest of the document is considered to be in the body of the script tag, and so is escaped.
The unit test HtmlParserTest.handlesKnownEmptyBlocks() will fail if a self-closing script tag is included in the String h.


int nextIndexOf(CharSequence seq) {
    // doesn't handle scanning for surrogates
    char startChar = seq.charAt(0);
    for (int offset = pos; offset < length; offset++) {
        // scan to first instance of startchar:
        if (startChar != input[offset])
            while(++offset < length && startChar != input[offset]);
        int i = offset + 1;
        int last = i + seq.length()-1;
        if (offset < length) {
            for (int j = 1; i < last && seq.charAt(j) == input[i]; i++, j++);
            if (i == last) // found full sequence
                return offset - pos;
        }
    }
    return -1;
}

int nextIndexOf(CharSequence seq) {
    // doesn't handle scanning for surrogates
    char startChar = seq.charAt(0);
    for (int offset = pos; offset < length; offset++) {
        // scan to first instance of startchar:
        if (startChar != input[offset])
            while(++offset < length && startChar != input[offset]);
        int i = offset + 1;
        int last = i + seq.length()-1;
        if (offset < length && last <= length) {
            for (int j = 1; i < last && seq.charAt(j) == input[i]; i++, j++);
            if (i == last) // found full sequence
                return offset - pos;
        }
    }
    return -1;
}

Jsoup gives the following error when trying to parse this HTML: https://gist.github.com/felipehummel/6122799
java.lang.ArrayIndexOutOfBoundsException: 8666
    at org.jsoup.parser.CharacterReader.nextIndexOf(CharacterReader.java:92)
    at org.jsoup.parser.CharacterReader.consumeTo(CharacterReader.java:112)
    at org.jsoup.parser.TokeniserState$67.read(TokeniserState.java:1789)
    at org.jsoup.parser.Tokeniser.read(Tokeniser.java:42)
    at org.jsoup.parser.TreeBuilder.runParser(TreeBuilder.java:47)
    at org.jsoup.parser.TreeBuilder.parse(TreeBuilder.java:41)
    at org.jsoup.parser.HtmlTreeBuilder.parse(HtmlTreeBuilder.java:37)
    at org.jsoup.parser.Parser.parse(Parser.java:90)
    at org.jsoup.Jsoup.parse(Jsoup.java:58)
...

The HTML is from a entry in a RSS feed. If I remove the line:
// ]]

or just the
]]
Then it parses the HTML nicely.
Does this syntax error should really throw an exception or it should be silently ignored?


public String html() {
    StringBuilder accum = new StringBuilder();
    html(accum);
    return accum.toString().trim();
}

public String html() {
    StringBuilder accum = new StringBuilder();
    html(accum);
    return getOutputSettings().prettyPrint() ? accum.toString().trim() : accum.toString();
}

Whitespaces are discared in Element.html() method

Hi,
I'm trying to make an exact copy of a document (changing just a couple of attributes and appending a few nodes) and the trim() inside the Element.html() is killing me.
I'm using Parsers.xml() and no prettyPrint.
I think this trim should be enabled for prettyPrint only.


static Document parseByteData(ByteBuffer byteData, String charsetName, String baseUri, Parser parser) {
    String docData;
    Document doc = null;
    if (charsetName == null) { // determine from meta. safe parse as UTF-8
        // look for <meta http-equiv="Content-Type" content="text/html;charset=gb2312"> or HTML5 <meta charset="gb2312">
        docData = Charset.forName(defaultCharset).decode(byteData).toString();
        doc = parser.parseInput(docData, baseUri);
        Element meta = doc.select("meta[http-equiv=content-type], meta[charset]").first();
        if (meta != null) { // if not found, will keep utf-8 as best attempt
            String foundCharset;
            if (meta.hasAttr("http-equiv")) {
                foundCharset = getCharsetFromContentType(meta.attr("content"));
                if (foundCharset == null && meta.hasAttr("charset")) {
                    try {
                        if (Charset.isSupported(meta.attr("charset"))) {
                            foundCharset = meta.attr("charset");
                        }
                    } catch (IllegalCharsetNameException e) {
                        foundCharset = null;
                    }
                }
            } else {
                foundCharset = meta.attr("charset");
            }

            if (foundCharset != null && foundCharset.length() != 0 && !foundCharset.equals(defaultCharset)) { // need to re-decode
                foundCharset = foundCharset.trim().replaceAll("[\"']", "");
                charsetName = foundCharset;
                byteData.rewind();
                docData = Charset.forName(foundCharset).decode(byteData).toString();
                doc = null;
            }
        }
    } else { // specified by content type header (or by user on file load)
        Validate.notEmpty(charsetName, "Must set charset arg to character set of file to parse. Set to null to attempt to detect from HTML");
        docData = Charset.forName(charsetName).decode(byteData).toString();
    }
    // UTF-8 BOM indicator. takes precedence over everything else. rarely used. re-decodes incase above decoded incorrectly
    if (docData.length() > 0 && docData.charAt(0) == 65279) {
        byteData.rewind();
        docData = Charset.forName(defaultCharset).decode(byteData).toString();
        docData = docData.substring(1);
        charsetName = defaultCharset;
    }
    if (doc == null) {
        doc = parser.parseInput(docData, baseUri);
        doc.outputSettings().charset(charsetName);
    }
    return doc;
}

static Document parseByteData(ByteBuffer byteData, String charsetName, String baseUri, Parser parser) {
    String docData;
    Document doc = null;
    if (charsetName == null) { // determine from meta. safe parse as UTF-8
        // look for <meta http-equiv="Content-Type" content="text/html;charset=gb2312"> or HTML5 <meta charset="gb2312">
        docData = Charset.forName(defaultCharset).decode(byteData).toString();
        doc = parser.parseInput(docData, baseUri);
        Element meta = doc.select("meta[http-equiv=content-type], meta[charset]").first();
        if (meta != null) { // if not found, will keep utf-8 as best attempt
            String foundCharset;
            if (meta.hasAttr("http-equiv")) {
                foundCharset = getCharsetFromContentType(meta.attr("content"));
                if (foundCharset == null && meta.hasAttr("charset")) {
                    try {
                        if (Charset.isSupported(meta.attr("charset"))) {
                            foundCharset = meta.attr("charset");
                        }
                    } catch (IllegalCharsetNameException e) {
                        foundCharset = null;
                    }
                }
            } else {
                foundCharset = meta.attr("charset");
            }

            if (foundCharset != null && foundCharset.length() != 0 && !foundCharset.equals(defaultCharset)) { // need to re-decode
                foundCharset = foundCharset.trim().replaceAll("[\"']", "");
                charsetName = foundCharset;
                byteData.rewind();
                docData = Charset.forName(foundCharset).decode(byteData).toString();
                doc = null;
            }
        }
    } else { // specified by content type header (or by user on file load)
        Validate.notEmpty(charsetName, "Must set charset arg to character set of file to parse. Set to null to attempt to detect from HTML");
        docData = Charset.forName(charsetName).decode(byteData).toString();
    }
    // UTF-8 BOM indicator. takes precedence over everything else. rarely used. re-decodes incase above decoded incorrectly
    if (docData.length() > 0 && docData.charAt(0) == 65279) {
        byteData.rewind();
        docData = Charset.forName(defaultCharset).decode(byteData).toString();
        docData = docData.substring(1);
        charsetName = defaultCharset;
        doc = null;
    }
    if (doc == null) {
        doc = parser.parseInput(docData, baseUri);
        doc.outputSettings().charset(charsetName);
    }
    return doc;
}

JSoup incorrectly moves content from the <head> section into <body> for sample URL

If you load the following URL:
http://jornutzon.sydneyoperahouse.com/home.htm

into:
http://try.jsoup.org/

then it will move the content from the "head" section into the "body" section. The URL
being parsed validates using the W3C validator:
http://validator.w3.org/check?uri=http%3A%2F%2Fjornutzon.sydneyoperahouse.com%2Fhome.htm&charset=%28detect+automatically%29&doctype=Inline&ss=1&group=0&user-agent=W3C_Validator%2F1.3+http%3A%2F%2Fvalidator.w3.org%2Fservices
We are using JSoup 1.7.2


public DocumentType(String name, String publicId, String systemId, String baseUri) {
    super(baseUri);

    Validate.notEmpty(name);
    attr("name", name);
    attr("publicId", publicId);
    attr("systemId", systemId);
}

public DocumentType(String name, String publicId, String systemId, String baseUri) {
    super(baseUri);

    attr("name", name);
    attr("publicId", publicId);
    attr("systemId", systemId);
}

"<!DOCTYPE>" IllegalArgumentException: String must not be empty

While this may be a contrived example, Jsoup.parse("<!DOCTYPE>") throws an exception, this was unexpected. Possibly related, a proper document with <!DOCTYPE> (no name) is generating corrupt html e.g. "<!DOCTYPE <html> ..." (missing right angle bracket on DOCTYPE.)
Spec says "When a DOCTYPE token is created, its name, public identifier, and system identifier must be marked as missing (which is a distinct state from the empty string), [...]"


public boolean equals(Object o) {
    if (this == o) return true;
    if (o == null || getClass() != o.getClass()) return false;
    if (!super.equals(o)) return false;

    Element element = (Element) o;

    return this == o;
}

public boolean equals(Object o) {
    if (this == o) return true;
    if (o == null || getClass() != o.getClass()) return false;
    if (!super.equals(o)) return false;

    Element element = (Element) o;

    return tag.equals(element.tag);
}

Element.hashCode() ignores the content text of the element.

Found this question on SO, OP was using Element.hashCode() and it wasn't woring right.
The problem is that when jsoup generates the hashCode of an Element, the content text of the element will be ignored, and the hashCode is generated only based on the attributes, and the hashCode of the parent Element.

Using the following HTML:

TODO: write content

Nothing here

Empty

Empty

And the following code:
String html = //HTML posted above

Document doc = Jsoup.parse(html);

Elements elements = doc.select("[style]");
for (Element e : elements) {
   System.out.println(e.hashCode());
}

It gives:
-148184373
-148184373
-1050420242
2013043377

I believe the hashCode should be different for the first two Elements, since the content is text is different. Or is this intended behaviour?


private static <E extends Element> Integer indexInList(Element search, List<E> elements) {
    Validate.notNull(search);
    Validate.notNull(elements);

    for (int i = 0; i < elements.size(); i++) {
        E element = elements.get(i);
        if (element.equals(search))
            return i;
    }
    return null;
}

private static <E extends Element> Integer indexInList(Element search, List<E> elements) {
    Validate.notNull(search);
    Validate.notNull(elements);

    for (int i = 0; i < elements.size(); i++) {
        E element = elements.get(i);
        if (element == search)
            return i;
    }
    return null;
}

Unexpected behavior in elementSiblingIndex

The documentation for elementSiblingIndex states "Get the list index of this element in its element sibling list. I.e. if this is the first element sibling, returns 0".
This would imply that if
n=myElem.elementSiblingIndex();

then
myElem.parent().children().get(n)==myElem.

However, this is not how elementSiblingIndex behaves. What is guaranteed is that
myElem.parent().children().get(n).equals(myElem).

For example, if both row 2 and row 5 of a table are

Cell1	Cell2

then the Element object associated with both rows will have the same elementSiblingIndex().


void resetInsertionMode() {
    boolean last = false;
    for (int pos = stack.size() -1; pos >= 0; pos--) {
        Element node = stack.get(pos);
        if (pos == 0) {
            last = true;
            node = contextElement;
        }
        String name = node.nodeName();
        if ("select".equals(name)) {
            transition(HtmlTreeBuilderState.InSelect);
            break; // frag
        } else if (("td".equals(name) || "td".equals(name) && !last)) {
            transition(HtmlTreeBuilderState.InCell);
            break;
        } else if ("tr".equals(name)) {
            transition(HtmlTreeBuilderState.InRow);
            break;
        } else if ("tbody".equals(name) || "thead".equals(name) || "tfoot".equals(name)) {
            transition(HtmlTreeBuilderState.InTableBody);
            break;
        } else if ("caption".equals(name)) {
            transition(HtmlTreeBuilderState.InCaption);
            break;
        } else if ("colgroup".equals(name)) {
            transition(HtmlTreeBuilderState.InColumnGroup);
            break; // frag
        } else if ("table".equals(name)) {
            transition(HtmlTreeBuilderState.InTable);
            break;
        } else if ("head".equals(name)) {
            transition(HtmlTreeBuilderState.InBody);
            break; // frag
        } else if ("body".equals(name)) {
            transition(HtmlTreeBuilderState.InBody);
            break;
        } else if ("frameset".equals(name)) {
            transition(HtmlTreeBuilderState.InFrameset);
            break; // frag
        } else if ("html".equals(name)) {
            transition(HtmlTreeBuilderState.BeforeHead);
            break; // frag
        } else if (last) {
            transition(HtmlTreeBuilderState.InBody);
            break; // frag
        }
    }
}

void resetInsertionMode() {
    boolean last = false;
    for (int pos = stack.size() -1; pos >= 0; pos--) {
        Element node = stack.get(pos);
        if (pos == 0) {
            last = true;
            node = contextElement;
        }
        String name = node.nodeName();
        if ("select".equals(name)) {
            transition(HtmlTreeBuilderState.InSelect);
            break; // frag
        } else if (("td".equals(name) || "th".equals(name) && !last)) {
            transition(HtmlTreeBuilderState.InCell);
            break;
        } else if ("tr".equals(name)) {
            transition(HtmlTreeBuilderState.InRow);
            break;
        } else if ("tbody".equals(name) || "thead".equals(name) || "tfoot".equals(name)) {
            transition(HtmlTreeBuilderState.InTableBody);
            break;
        } else if ("caption".equals(name)) {
            transition(HtmlTreeBuilderState.InCaption);
            break;
        } else if ("colgroup".equals(name)) {
            transition(HtmlTreeBuilderState.InColumnGroup);
            break; // frag
        } else if ("table".equals(name)) {
            transition(HtmlTreeBuilderState.InTable);
            break;
        } else if ("head".equals(name)) {
            transition(HtmlTreeBuilderState.InBody);
            break; // frag
        } else if ("body".equals(name)) {
            transition(HtmlTreeBuilderState.InBody);
            break;
        } else if ("frameset".equals(name)) {
            transition(HtmlTreeBuilderState.InFrameset);
            break; // frag
        } else if ("html".equals(name)) {
            transition(HtmlTreeBuilderState.BeforeHead);
            break; // frag
        } else if (last) {
            transition(HtmlTreeBuilderState.InBody);
            break; // frag
        }
    }
}

An issue where a table nested within a TH cell would parse to an incorrect tree


void processResponseHeaders(Map<String, List<String>> resHeaders) {
    for (Map.Entry<String, List<String>> entry : resHeaders.entrySet()) {
        String name = entry.getKey();
        if (name == null)
            continue; // http/1.1 line

        List<String> values = entry.getValue();
        if (name.equalsIgnoreCase("Set-Cookie")) {
            for (String value : values) {
                if (value == null)
                    continue;
                TokenQueue cd = new TokenQueue(value);
                String cookieName = cd.chompTo("=").trim();
                String cookieVal = cd.consumeTo(";").trim();
                // ignores path, date, domain, validateTLSCertificates et al. req'd?
                // name not blank, value not null
                if (cookieName.length() > 0)
                    cookie(cookieName, cookieVal);
            }
        } else { // combine same header names with comma: http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
            if (!values.isEmpty())
                header(name, values.get(0));
        }
    }
}

void processResponseHeaders(Map<String, List<String>> resHeaders) {
    for (Map.Entry<String, List<String>> entry : resHeaders.entrySet()) {
        String name = entry.getKey();
        if (name == null)
            continue; // http/1.1 line

        List<String> values = entry.getValue();
        if (name.equalsIgnoreCase("Set-Cookie")) {
            for (String value : values) {
                if (value == null)
                    continue;
                TokenQueue cd = new TokenQueue(value);
                String cookieName = cd.chompTo("=").trim();
                String cookieVal = cd.consumeTo(";").trim();
                // ignores path, date, domain, validateTLSCertificates et al. req'd?
                // name not blank, value not null
                if (cookieName.length() > 0)
                    cookie(cookieName, cookieVal);
            }
        } else { // combine same header names with comma: http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
            if (values.size() == 1)
                header(name, values.get(0));
            else if (values.size() > 1) {
                StringBuilder accum = new StringBuilder();
                for (int i = 0; i < values.size(); i++) {
                    final String val = values.get(i);
                    if (i != 0)
                        accum.append(", ");
                    accum.append(val);
                }
                header(name, accum.toString());
            }
        }
    }
}

A small bug for duplicate tuple in response header

for response headers have duplicate tuple，
in this case
X-Powered-By:PHP/5.2.8
X-Powered-By:ASP.NET
Jsoup can only get the second one
if I run header（“X-powered-by”）
I got Asp.NET
URL：http://01pt.com/
Cache-Control:no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Content-Encoding:gzip
Content-Length:16224
Content-Type:text/html;charset=gb2312
Date:Thu, 27 Aug 2015 09:22:40 GMT
Expires:Thu, 19 Nov 1981 08:52:00 GMT
Pragma:no-cache
Server:Microsoft-IIS/7.5
Vary:Accept-Encoding
X-Powered-By:PHP/5.2.8
X-Powered-By:ASP.NET
The bug is because
if (!values.isEmpty()) header(name, values.get(0));
I change it to
if (!values.isEmpty()) {
String val = "";
for(String str: values) {
val = val.concat(str).concat(" ");
                    }
                    header(name, val);
              }

then I am able to get “PHP/5.2.8 ASP.NET” when I run header（“X-powered-by”）
void processResponseHeaders(Map<String, List> resHeaders) {
for (Map.Entry<String, List> entry : resHeaders.entrySet()) {
String name = entry.getKey();
if (name == null)
continue; // http/1.1 line
            List<String> values = entry.getValue();
            if (name.equalsIgnoreCase("Set-Cookie")) {
                for (String value : values) {
                    if (value == null)
                        continue;
                    TokenQueue cd = new TokenQueue(value);
                    String cookieName = cd.chompTo("=").trim();
                    String cookieVal = cd.consumeTo(";").trim();
                    // ignores path, date, domain, validateTLSCertificates et al. req'd?
                    // name not blank, value not null
                    if (cookieName.length() > 0)
                        cookie(cookieName, cookieVal);
                }
            } else { // only take the first instance of each header
                if (!values.isEmpty())
                    header(name, values.get(0));
            }
        }
    }


protected void addChildren(int index, Node... children) {
    Validate.noNullElements(children);
    ensureChildNodes();
    for (int i = children.length - 1; i >= 0; i--) {
        Node in = children[i];
        reparentChild(in);
        childNodes.add(index, in);
    }
    reindexChildren(index);
}

protected void addChildren(int index, Node... children) {
    Validate.noNullElements(children);
    ensureChildNodes();
    for (int i = children.length - 1; i >= 0; i--) {
        Node in = children[i];
        reparentChild(in);
        childNodes.add(index, in);
        reindexChildren(index);
    }
}

When using org.jsoup.nodes.Element.insertChildren(int, Collection<? extends Node>) to move (more than one!) child-elements from one parent-element to the same parent, but different index then it produces wrong results.
The problem is that the first Element's 'move' leaves the siblingIndex unchanged and then the second 'move' removes a wrong element and produces some crap. Maybe calling reindexChildren() inside the loop in addChildren() fixes this.
Version 1.8.3.
Workaround: call remove() on the elements before passing them to insertChildren()
Easy Test Case:
    @Test
    public void mustCorrectlyMoveChildrenInsideOneParentElement() {

        Document doc = new Document( "" );
        Element body = doc.appendElement( "body" );
        body.appendElement( "div1" );
        body.appendElement( "div2" );
        Element div3 = body.appendElement( "div3" );
        Element div4 = body.appendElement( "div4" );

        ArrayList<Element> toMove = new ArrayList<Element>() {
            {
                add( div3 );
                add( div4 );
            }
        };

        body.insertChildren( 0, toMove );

        String result = doc.toString().replaceAll( "\\s+", "" );
        assertEquals( "<body><div3></div3><div4></div4><div1></div1><div2></div2></body>", result );

    }


private Attribute parseAttribute() {
    tq.consumeWhitespace();
    String key = tq.consumeAttributeKey();
    String value = "";
    tq.consumeWhitespace();
    if (tq.matchChomp("=")) {
        tq.consumeWhitespace();

        if (tq.matchChomp(SQ)) {
            value = tq.chompTo(SQ);
        } else if (tq.matchChomp(DQ)) {
            value = tq.chompTo(DQ);
        } else {
            StringBuilder valueAccum = new StringBuilder();
            // no ' or " to look for, so scan to end tag or space (or end of stream)
            while (!tq.matchesAny("<", "/>", ">") && !tq.matchesWhitespace() && !tq.isEmpty()) {
                valueAccum.append(tq.consume());
            }
            value = valueAccum.toString();
        }
        tq.consumeWhitespace();
    }
    if (key.length() != 0)
        return Attribute.createFromEncoded(key, value);
    else {
        tq.consume();

        return null;
    }
}

private Attribute parseAttribute() {
    tq.consumeWhitespace();
    String key = tq.consumeAttributeKey();
    String value = "";
    tq.consumeWhitespace();
    if (tq.matchChomp("=")) {
        tq.consumeWhitespace();

        if (tq.matchChomp(SQ)) {
            value = tq.chompTo(SQ);
        } else if (tq.matchChomp(DQ)) {
            value = tq.chompTo(DQ);
        } else {
            StringBuilder valueAccum = new StringBuilder();
            // no ' or " to look for, so scan to end tag or space (or end of stream)
            while (!tq.matchesAny("<", "/>", ">") && !tq.matchesWhitespace() && !tq.isEmpty()) {
                valueAccum.append(tq.consume());
            }
            value = valueAccum.toString();
        }
        tq.consumeWhitespace();
    }
    if (key.length() != 0)
        return Attribute.createFromEncoded(key, value);
    else {
        if (value.length() == 0) // no key, no val; unknown char, keep popping so not get stuck
            tq.advance();

        return null;
    }
}

StringIndexOutOfBoundsException when testing whether String content is valid HTML

If I try to parse a tag with an equals sign (an empty attribute) but without any single or double quotes around an attribute value, then I get a StringIndexOutOfBoundsException.  The stack trace is pasted below.
An example String would be "<a =a"
The following JUnit test case should not throw a StringIndexOutOfBoundsException:
import static org.junit.Assert.assertTrue;
import org.jsoup.Jsoup;
import org.jsoup.safety.Whitelist;
import org.junit.Test;
public class BadAttributeTest {
@test
public void aTagWithABadAttributeIsValid() throws Exception {
assertTrue(Jsoup.isValid("<a =a", Whitelist.relaxed()));
}
}
java.lang.StringIndexOutOfBoundsException: String index out of range: 13
at java.lang.String.charAt(String.java:686)
at org.jsoup.parser.TokenQueue.consume(TokenQueue.java:130)
at org.jsoup.parser.Parser.parseAttribute(Parser.java:207)
at org.jsoup.parser.Parser.parseStartTag(Parser.java:142)
at org.jsoup.parser.Parser.parse(Parser.java:91)
at org.jsoup.parser.Parser.parseBodyFragment(Parser.java:64)
at org.jsoup.Jsoup.parseBodyFragment(Jsoup.java:99)
at org.jsoup.Jsoup.isValid(Jsoup.java:155)