Split ArrayConverter list elements on whitespace, quotes and the delimiter only

src/main/java/org/apache/commons/beanutils2/converters/ArrayConverter.java

@@ -64,11 +64,12 @@
  * </li>
  * </ul>
  *
- * <h2>Parsing Delimited Lists</h2> This implementation can convert a delimited list in {@code String} format into an array of the appropriate type. By default,
- * it uses a comma as the delimiter but the following methods can be used to configure parsing:
+ * <h2>Parsing Delimited Lists</h2> This implementation can convert a delimited list in {@code String} format into an array of the appropriate type. The
+ * String is split on the delimiter and on whitespace; every other character is kept as part of an element, and elements may be quoted with single or double
+ * quotes to protect embedded whitespace or delimiters. By default, it uses a comma as the delimiter but the following method can be used to configure
+ * parsing:
  * <ul>
  * <li>{@code setDelimiter(char)} - allows the character used as the delimiter to be configured [default is a comma].</li>
- * <li>{@code setAllowedChars(char[])} - adds additional characters (to the default alphabetic/numeric) to those considered to be valid token characters.</li>
  * </ul>
  *
  * <h2>Multi Dimensional Arrays</h2> It is possible to convert a {@code String} to multi-dimensional arrays by using {@link ArrayConverter} as the element
@@ -89,11 +90,8 @@
  * // Construct a "Matrix" Converter which converts arrays of integer arrays using
  * // the preceding ArrayConverter as the element Converter.
  * // Uses a semicolon (i.e. ";") as the delimiter to separate the different sets of numbers.
- * // Also the delimiter used by the first ArrayConverter needs to be added to the
- * // "allowed characters" for this one.
  * ArrayConverter matrixConverter = new ArrayConverter(int[][].class, arrayConverter);
  * matrixConverter.setDelimiter(';');
- * matrixConverter.setAllowedChars(new char[] { ',' });
  *
  * // Do the Conversion
  * String matrixString = "11,12,13 ; 21,22,23 ; 31,32,33 ; 41,42,43";
@@ -310,10 +308,10 @@ protected Class<C> getDefaultType() {
      * according to the following rules.
      * </p>
      * <ul>
-     * <li>The string is expected to be a comma-separated list of values.</li>
+     * <li>The string is split on the delimiter [default is a comma] and on whitespace; every other character is kept as part of an element.</li>
      * <li>The string may optionally have matching '{' and '}' delimiters around the list.</li>
-     * <li>Whitespace before and after each element is stripped.</li>
-     * <li>Elements in the list may be delimited by single or double quotes. Within a quoted elements, the normal Java escape sequences are valid.</li>
+     * <li>Elements in the list may be delimited by single or double quotes. A quoted element may contain whitespace and the delimiter, and within a quoted
+     * element the normal Java escape sequences are valid.</li>
      * </ul>
      *
      * @param value String value to be parsed
@@ -335,17 +333,18 @@ private List<String> parseElements(String value) {
         final String typeName = toString(String.class);
         try {
 
-            // Set up a StreamTokenizer on the characters in this String
+            // Set up a StreamTokenizer on the characters in this String. Every character is part of a token except whitespace, the quote characters and the
+            // delimiter, so elements are only split on those. The default syntax table would also split on any other non-alphanumeric character and treat
+            // '/' as a comment start, silently dropping the rest of the input.
             final StreamTokenizer st = new StreamTokenizer(new StringReader(value));
-            st.whitespaceChars(delimiter, delimiter); // Set the delimiters
-            st.ordinaryChars('0', '9'); // Needed to turn off numeric flag
-            st.wordChars('0', '9'); // Needed to make part of tokens
-            for (final char allowedChar : allowedChars) {
-                st.ordinaryChars(allowedChar, allowedChar);
-                st.wordChars(allowedChar, allowedChar);
-            }
-
-            // Split comma-delimited tokens into a List
+            st.resetSyntax();
+            st.wordChars(0, 255); // Everything is part of a token...
+            st.whitespaceChars(0, ' '); // ...except whitespace...
+            st.quoteChar('"'); // ...quoted elements, which may contain whitespace and the delimiter...
+            st.quoteChar('\'');
+            st.whitespaceChars(delimiter, delimiter); // ...and the delimiter, which separates elements.
+
+            // Split the tokens into a List
             List<String> list = null;
             while (true) {
                 final int ttype = st.nextToken();
@@ -382,7 +381,10 @@ private List<String> parseElements(String value) {
      * Sets the allowed characters to be used for parsing a delimited String.
      *
      * @param allowedChars Characters which are to be considered as part of the tokens when parsing a delimited String [default is '.' and '-']
+     * @deprecated Since 1.12.0: No longer has any effect: every character apart from whitespace, the delimiter and the quote characters is kept
+     * as part of an element.
      */
+    @Deprecated
     public void setAllowedChars(final char[] allowedChars) {
         this.allowedChars = Objects.requireNonNull(allowedChars, "allowedChars").clone();
     }

src/test/java/org/apache/commons/beanutils2/bugs/Jira359Test.java

@@ -105,11 +105,9 @@ void testBeanUtilsSetProperty_DefaultConvertStringToArray_WithColonValue() throw
         final SimplePojoData simplePojo = new SimplePojoData();
         BeanUtils.setProperty(simplePojo, "jcrMixinTypes", "mix:rereferencible,mix:simple");
         showArray("Default WithColonValue", simplePojo.getJcrMixinTypes());
-        assertEquals(4, simplePojo.getJcrMixinTypes().length, "array size");
-        assertEquals("mix", simplePojo.getJcrMixinTypes()[0]);
-        assertEquals("rereferencible", simplePojo.getJcrMixinTypes()[1]);
-        assertEquals("mix", simplePojo.getJcrMixinTypes()[2]);
-        assertEquals("simple", simplePojo.getJcrMixinTypes()[3]);
+        assertEquals(2, simplePojo.getJcrMixinTypes().length, "array size");
+        assertEquals("mix:rereferencible", simplePojo.getJcrMixinTypes()[0]);
+        assertEquals("mix:simple", simplePojo.getJcrMixinTypes()[1]);
     }
 
     /**

src/test/java/org/apache/commons/beanutils2/converters/ArrayConverterTest.java

@@ -163,21 +163,14 @@ void testErrors() {
         assertThrows(NullPointerException.class, () -> new ArrayConverter(int[].class, null));
     }
 
+    /**
+     * A forward slash must be kept as part of an element instead of starting a comment and dropping the rest of the input.
+     */
+    @Test
     void testForwardSlashSeparator() {
         final String value = "first/value,second/value";
         final ArrayConverter<String[]> converter = new ArrayConverter<>(String[].class, new StringConverter());
-        // test forward slash not allowed (the default)
-        String[] result = converter.convert(String[].class, value);
-        assertNotNull(result, "result.null");
-        assertEquals(4, result.length, "result.length");
-        assertEquals("first", result[0], "result[0]");
-        assertEquals("value", result[1], "result[1]");
-        assertEquals("second", result[2], "result[2]");
-        assertEquals("value", result[3], "result[3]");
-        // configure the converter to allow forward slash
-        converter.setAllowedChars(new char[] { '.', '-', '/' });
-        // test forward slash allowed
-        result = converter.convert(String[].class, value);
+        final String[] result = converter.convert(String[].class, value);
         assertNotNull(result, "result.null");
         assertEquals(2, result.length, "result.length");
         assertEquals("first/value", result[0], "result[0]");
@@ -248,11 +241,8 @@ void testTheMatrix() {
         // Construct a "Matrix" Converter which converts arrays of integer arrays using
         // the first (int[]) Converter as the element Converter.
         // Uses a semicolon (i.e. ";") as the delimiter to separate the different sets of numbers.
-        // Also the delimiter for the above array Converter needs to be added to this
-        // array Converter's "allowed characters"
         final ArrayConverter matrixConverter = new ArrayConverter(int[][].class, arrayConverter);
         matrixConverter.setDelimiter(';');
-        matrixConverter.setAllowedChars(new char[] { ',' });
         // Do the Conversion
         final Object result = matrixConverter.convert(int[][].class, matrixString);
         // Check it actually worked OK
@@ -270,24 +260,13 @@ void testTheMatrix() {
     }
 
     /**
-     * Test for BEANUTILS-302 throwing a NPE when underscore used.
+     * Test for BEANUTILS-302 throwing a NPE when underscore used. The underscore is kept as part of the element.
      */
     @Test
     void testUnderscore_BEANUTILS_302() {
         final String value = "first_value,second_value";
-        final ArrayConverter<String[]> converter = new ArrayConverter(String[].class, new StringConverter());
-        // test underscore not allowed (the default)
-        String[] result = converter.convert(String[].class, value);
-        assertNotNull(result, "result.null");
-        assertEquals(4, result.length, "result.length");
-        assertEquals("first", result[0], "result[0]");
-        assertEquals("value", result[1], "result[1]");
-        assertEquals("second", result[2], "result[2]");
-        assertEquals("value", result[3], "result[3]");
-        // configure the converter to allow underscore
-        converter.setAllowedChars(new char[] { '.', '-', '_' });
-        // test underscore allowed
-        result = converter.convert(String[].class, value);
+        final ArrayConverter<String[]> converter = new ArrayConverter<>(String[].class, new StringConverter());
+        final String[] result = converter.convert(String[].class, value);
         assertNotNull(result, "result.null");
         assertEquals(2, result.length, "result.length");
         assertEquals("first_value", result[0], "result[0]");