DcmCharString: add some support for multi-byte characters

- add DcmCharString::getVM(), getOFString() and putOFStringAtPos(), which handle multi-byte charsets
- DcmByteString::containsExtendedCharacters():
  add check for ESCAPE characters (only allowed in code extensions)
- removed obsolete DcmCharString::containsExtendedCharacters()

Author: mrbean-bremen

dcmdata/include/dcmtk/dcmdata/dcbytstr.h

@@ -274,20 +274,19 @@ class DCMTK_DCMDATA_EXPORT DcmByteString: public DcmElement
      */
     virtual OFCondition verify(const OFBool autocorrect = OFFalse);
 
-    /** check if this element contains non-ASCII characters. Please note that this check
-     *  is pretty simple and only works for single-byte character sets that do include
-     *  the 7-bit ASCII codes, e.g. for the ISO 8859 family. In other words: All character
-     *  codes below 128 are considered to be ASCII codes and all others are considered to
-     *  be non-ASCII.
+    /** check if this element contains non-ASCII characters.
+     *  This works by checking for any byte values above 127, which works for any
+     *  single-byte code and for single-value multi-byte codes, and for ESC characters,
+     *  which will mean that a code extension is used.
      *  @param checkAllStrings if true, also check elements with string values not affected
      *    by SpecificCharacterSet (0008,0005). By default, only check PN, LO, LT, SH, ST,
-     *    UC and UT, i.e. none of the derived VR classes.
+     *    UC and UT.
      *  @return true if element contains non-ASCII characters, false otherwise
      */
     virtual OFBool containsExtendedCharacters(const OFBool checkAllStrings = OFFalse);
 
     /** check if this element is affected by SpecificCharacterSet
-     *  @return always returns false since none of the derived VR classes is affected by
+     *  @return returns false, overwritten by derived VR classes that are affected by
      *    the SpecificCharacterSet (0008,0005) element
      */
     virtual OFBool isAffectedBySpecificCharacterSet() const;
@@ -379,6 +378,20 @@ class DCMTK_DCMDATA_EXPORT DcmByteString: public DcmElement
      */
     virtual OFCondition makeMachineByteString(const Uint32 length = 0);
 
+    /** check if the VR supports more than one value.
+     * @return OFTrue
+     */
+    virtual OFBool supportsMultiValue() const { return OFTrue; }
+
+    /** find the start index of the next component.
+     * @param str pointer to the string value to be searched
+     * @param len the length of @a str
+     * @param start the start character index for the search
+     * @param charSet the value of Specific Character Set; not used
+     * @return the index of the next value, or OFString_npos if none exists.
+     */
+    virtual size_t findNextValuePosition(const char* str, size_t len, size_t start, const OFString& charSet) const;
+
     /** convert currently stored string value to DICOM representation.
      *  It removes trailing spaces apart from a possibly required single padding
      *  character (in case of odd string length).
@@ -418,13 +431,23 @@ class DCMTK_DCMDATA_EXPORT DcmByteString: public DcmElement
      */
     void setNonSignificantChars(const OFString &characters) { nonSignificantChars = characters; }
 
+    /** set element value at a specific value position in the given character string,
+     * considering the specific character set for finding the position, if given.
+     *  @param stringVal input character string (possibly multi-valued)
+     *  @param pos position (0..vm) where the value should be inserted
+     *  @param charSet the value of the Specific Character Set
+     *  @return status, EC_Normal if successful, an error code otherwise
+     */
+    OFCondition putOFStringAtPosWithCharset(const OFString& stringVal,
+                                            const unsigned long pos,
+                                            const OFString& charSet);
+
     /* --- static helper functions --- */
 
     /** check if a given character string contains non-ASCII characters.
-     *  Please note that this check is pretty simple and only works for single-byte character
-     *  sets that do include the 7-bit ASCII codes, e.g. for the ISO 8859 family. In other
-     *  words: All character codes below 128 are considered to be ASCII codes and all others
-     *  are considered to be non-ASCII.
+     *  This works by checking for any byte values above 127, which works for any
+     *  single-byte code and for single-value multi-byte codes, and for ESC characters,
+     *  which will mean that a code extension is used.
      *  @param stringVal character string to be checked
      *  @param stringLen length of the string (number of characters without the trailing
      *    NULL byte)

dcmdata/include/dcmtk/dcmdata/dcchrstr.h

@@ -107,17 +107,31 @@ class DCMTK_DCMDATA_EXPORT DcmCharString
      */
     virtual OFCondition verify(const OFBool autocorrect = OFFalse);
 
-    /** check if this element contains non-ASCII characters. Please note that this check
-     *  is pretty simple and only works for single-byte character sets that do include
-     *  the 7-bit ASCII codes, e.g. for the ISO 8859 family. In other words: All character
-     *  codes below 128 are considered to be ASCII codes and all others are considered to
-     *  be non-ASCII.
-     *  @param checkAllStrings not used in this class
-     *  @return true if element contains non-ASCII characters, false otherwise
+    /** get value multiplicity
+     *  @return number of string components (separated by a backslash)
      */
-    virtual OFBool containsExtendedCharacters(const OFBool checkAllStrings = OFFalse);
+    virtual unsigned long getVM();
 
-    /** check if this element is affected by SpecificCharacterSet
+    /** get a copy of a particular string component
+     *  @param stringVal variable in which the result value is stored
+     *  @param pos index of the value in case of multi-valued elements (0..vm-1)
+     *  @param normalize not used since string normalization depends on value representation
+     *  @return status, EC_Normal if successful, an error code otherwise
+     */
+    virtual OFCondition getOFString(OFString &stringVal,
+                                    const unsigned long pos,
+                                    OFBool normalize = OFTrue);
+
+
+    /** set element value at specific VM position in the given character string.
+     *  @param stringVal input character string (possibly multi-valued)
+     *  @param pos position (0..vm) where the value should be inserted
+     *  @return status, EC_Normal if successful, an error code otherwise
+     */
+    virtual OFCondition putOFStringAtPos(const OFString& stringVal,
+                                         const unsigned long pos = 0);
+
+  /** check if this element is affected by SpecificCharacterSet
      *  @return always returns true since all derived VR classes are affected by the
      *    SpecificCharacterSet (0008,0005) element
      */
@@ -169,6 +183,15 @@ class DCMTK_DCMDATA_EXPORT DcmCharString
      */
     virtual const OFString& getDelimiterChars() const;
 
+    /** find the start index of the next value in a multi-valued attribute.
+     * @param str pointer to the string value to be searched
+     * @param len the length of @a str
+     * @param start the start character index for the search
+     * @param charSet the value of Specific Character Set; if not set, single-byte encoding is assumed
+     * @return the index of the next value, or OFString_npos if none exists.
+     */
+    virtual size_t findNextValuePosition(const char* str, size_t len, size_t start, const OFString& charSet) const;
+
 };
 
 

dcmdata/include/dcmtk/dcmdata/dcitem.h

@@ -317,11 +317,10 @@ class DCMTK_DCMDATA_EXPORT DcmItem
      */
     virtual OFBool containsUnknownVR() const;
 
-    /** check if this object contains non-ASCII characters at any nesting level. Please note
-     *  that this check is pretty simple and only works for single-byte character sets that
-     *  do include the 7-bit ASCII codes, e.g. for the ISO 8859 family. In other words: All
-     *  character codes below 128 are considered to be ASCII codes and all others are
-     *  considered to be non-ASCII.
+    /** check if this object contains non-ASCII characters.
+     *  This works by checking for any byte values above 127, which works for any
+     *  single-byte code and for single-value multi-byte codes, and for ESC characters,
+     *  which will mean that a code extension is used.
      *  @param checkAllStrings if true, also check elements with string values not affected
      *    by SpecificCharacterSet (0008,0005). By default, only check PN, LO, LT, SH, ST,
      *    UC and UT.

dcmdata/include/dcmtk/dcmdata/dcsequen.h

@@ -313,11 +313,10 @@ class DCMTK_DCMDATA_EXPORT DcmSequenceOfItems : public DcmElement
      */
     virtual OFBool containsUnknownVR() const;
 
-    /** check if this object contains non-ASCII characters at any nesting level. Please note
-     *  that this check is pretty simple and only works for single-byte character sets that
-     *  do include the 7-bit ASCII codes, e.g. for the ISO 8859 family. In other words: All
-     *  character codes below 128 are considered to be ASCII codes and all others are
-     *  considered to be non-ASCII.
+    /** check if this object contains non-ASCII characters.
+     *  This works by checking for any byte values above 127, which works for any
+     *  single-byte code and for single-value multi-byte codes, and for ESC characters,
+     *  which will mean that a code extension is used.
      *  @param checkAllStrings if true, also check elements with string values not affected
      *    by SpecificCharacterSet (0008,0005). By default, only check PN, LO, LT, SH, ST,
      *    UC and UT.

dcmdata/include/dcmtk/dcmdata/dcspchrs.h

@@ -232,6 +232,15 @@ class DCMTK_DCMDATA_EXPORT DcmSpecificCharacterSet
      */
     static size_t countCharactersInUTF8String(const OFString &utf8String);
 
+    /** check whether the given Specific Character Set value belongs to a multi-byte
+     * character set that is only allowed as a single value
+     * in SpecificCharacterSet and that may contain bytes that look like ASCII (e.g.
+     * with the highest bit cleared), but are part of a multi-byte non-ASCII character.
+     * This currently includes only Chinese encodings, as in UTF-8 (ISO_IR 192) any byte
+     * belonging to a non-ASCII character has the highest bit set
+     */
+    static OFBool isNonAsciiConformMultiByteSingleValueCharacterSet(const OFString& charset);
+
 
   protected:
 

dcmdata/include/dcmtk/dcmdata/dcvrlt.h

@@ -114,13 +114,6 @@ class DCMTK_DCMDATA_EXPORT DcmLongText
     virtual OFCondition checkValue(const OFString &vm = "",
                                    const OFBool oldFormat = OFFalse);
 
-    /** get the value multiplicity.
-     *  Since the backslash "\" is not regarded as a separator the value
-     *  multiplicity is always 1.
-     *  @return value multiplicity of the currently stored value
-     */
-    virtual unsigned long getVM();
-
     /** get a copy of a particular string component
      *  @param stringVal variable in which the result value is stored
      *  @param pos index of the value in case of multi-valued elements (0..vm-1)
@@ -151,6 +144,14 @@ class DCMTK_DCMDATA_EXPORT DcmLongText
      */
     static OFCondition checkStringValue(const OFString &value,
                                         const OFString &charset = "");
+
+  protected:
+    /** check if the VR supports more than one value.
+     *  Since the backslash "\" is not regarded as a separator,
+     *  multiple values cannot be encoded.
+     *  @return OFFalse
+     */
+    virtual OFBool supportsMultiValue() const { return OFFalse; };
 };
 
 

dcmdata/include/dcmtk/dcmdata/dcvrst.h

@@ -115,13 +115,6 @@ class DCMTK_DCMDATA_EXPORT DcmShortText
     virtual OFCondition checkValue(const OFString &vm = "",
                                    const OFBool oldFormat = OFFalse);
 
-    /** get the value multiplicity.
-     *  Since the backslash "\" is not regarded as a separator the value
-     *  multiplicity is always 1.
-     *  @return value multiplicity of the currently stored value
-     */
-    virtual unsigned long getVM();
-
     /** get a copy of a particular string component
      *  @param stringVal variable in which the result value is stored
      *  @param pos index of the value in case of multi-valued elements (0..vm-1)
@@ -152,6 +145,13 @@ class DCMTK_DCMDATA_EXPORT DcmShortText
      */
     static OFCondition checkStringValue(const OFString &value,
                                         const OFString &charset = "");
+protected:
+  /** check if the VR supports more than one value.
+   *  Since the backslash "\" is not regarded as a separator,
+   *  multiple values cannot be encoded.
+   *  @return OFFalse
+   */
+  virtual OFBool supportsMultiValue() const { return OFFalse; };
 };
 
 

dcmdata/include/dcmtk/dcmdata/dcvrut.h

@@ -117,13 +117,6 @@ class DCMTK_DCMDATA_EXPORT DcmUnlimitedText
     virtual OFCondition checkValue(const OFString &vm = "",
                                    const OFBool oldFormat = OFFalse);
 
-    /** get the value multiplicity.
-     *  Since the backslash "\" is not regarded as a separator the value
-     *  multiplicity is always 1.
-     *  @return value multiplicity of the currently stored value
-     */
-    virtual unsigned long getVM();
-
     /** get a copy of a particular string component
      *  @param stringVal variable in which the result value is stored
      *  @param pos index of the value in case of multi-valued elements (0..vm-1)
@@ -154,6 +147,13 @@ class DCMTK_DCMDATA_EXPORT DcmUnlimitedText
      */
     static OFCondition checkStringValue(const OFString &value,
                                         const OFString &charset = "");
+protected:
+  /** check if the VR supports more than one value.
+   *  Since the backslash "\" is not regarded as a separator,
+   *  multiple values cannot be encoded.
+   *  @return OFFalse
+   */
+  virtual OFBool supportsMultiValue() const { return OFFalse; };
 };
 
 

dcmdata/libsrc/dcbytstr.cc

@@ -467,9 +467,8 @@ OFCondition DcmByteString::putString(const char *stringVal,
     return errorFlag;
 }
 
-
-OFCondition DcmByteString::putOFStringAtPos(const OFString& stringVal,
-                                            const unsigned long pos)
+OFCondition DcmByteString::putOFStringAtPosWithCharset(const OFString& stringVal, const unsigned long pos,
+    const OFString& charSet)
 {
     OFCondition result;
     // Get old value
@@ -505,46 +504,42 @@ OFCondition DcmByteString::putOFStringAtPos(const OFString& stringVal,
             // First value is set: Replace old value with new value
             else
             {
-                rightPos = str.find_first_of('\\', 0);
-                str = str.replace(0, rightPos, stringVal);
+                rightPos = findNextValuePosition(str.c_str(), str.length(), 0, charSet);
+                str = str.replace(0, rightPos - 1, stringVal);
             }
             return putOFStringArray(str);
         }
 
         // 3rd case: New value should be inserted somewhere in the middle
         size_t leftPos = 0;
         size_t vmPos = 0;
+        size_t strLen = str.length();
         // First, find the correct position, and then insert / replace new value
         do
         {
             // Step from value to value by looking for delimiters.
-            // Special handling first search (start looking at position 0 instead of 1)
-            if (vmPos == 0) leftPos = str.find('\\', 0);
-            else leftPos = str.find('\\', leftPos + 1 );
-            // leftPos = str.find('\\', leftPos == 0 ? 0 : leftPos +1);
+            leftPos = findNextValuePosition(str.c_str(), strLen, leftPos, charSet);
             if (leftPos != OFString_npos)
-            {
                 vmPos++;
-            }
         }
         while ( (leftPos != OFString_npos) && (vmPos != pos) );
-        rightPos = str.find_first_of('\\', leftPos+1);
-        if (rightPos == OFString_npos) rightPos = str.length();
+        rightPos = findNextValuePosition(str.c_str(), strLen, leftPos, charSet);
+        if (rightPos == OFString_npos) rightPos = strLen + 1;
 
         // If we do not have an old value of size 1 or we have an empty value
         if (rightPos - leftPos == 1)
         {
             // Empty value
             if (str.at(leftPos) == '\\')
-                str = str.insert(rightPos, stringVal);
+                str = str.insert(leftPos, stringVal);
             // Old value (length 1)
             else
                 str = str.replace(leftPos, 1, stringVal);
         }
         // Otherwise replace existing old value (length > 1)
         else
         {
-            str = str.replace(leftPos+1, rightPos - leftPos - 1, stringVal);
+            str = str.replace(leftPos, rightPos - leftPos - 1, stringVal);
         }
         // Finally re-insert all values include new value
         result = putOFStringArray( str );
@@ -553,6 +548,28 @@ OFCondition DcmByteString::putOFStringAtPos(const OFString& stringVal,
 }
 
 
+
+OFCondition DcmByteString::putOFStringAtPos(const OFString& stringVal,
+                                            const unsigned long pos)
+{
+    return putOFStringAtPosWithCharset(stringVal, pos, "");
+}
+
+
+// ********************************
+
+
+size_t DcmByteString::findNextValuePosition(const char* str, size_t len, size_t start, const OFString& /*charSet*/) const
+{
+    const char *p = str + start;
+    for (size_t i = start; i < len; ++i)
+    {
+        if (*p++ == '\\')
+            return i + 1;
+    }
+    return OFString_npos;
+}
+
 // ********************************
 
 
@@ -770,7 +787,7 @@ OFBool DcmByteString::containsExtendedCharacters(const OFBool checkAllStrings)
     OFBool result = OFFalse;
     /* only check if parameter is true since derived VRs are not affected
        by the attribute SpecificCharacterSet (0008,0005) */
-    if (checkAllStrings)
+    if (checkAllStrings || isAffectedBySpecificCharacterSet())
     {
         char *str = NULL;
         Uint32 len = 0;
@@ -876,10 +893,10 @@ OFBool DcmByteString::containsExtendedCharacters(const char *stringVal,
 {
     if (stringVal != NULL)
     {
-        for (size_t i = stringLen; i != 0; --i)
+        for (size_t i = stringLen; i != 0; --i, ++stringVal)
         {
-            /* check for 8 bit characters */
-            if (OFstatic_cast(unsigned char, *stringVal++) > 127)
+            /* check for 8 bit and Escape characters */
+            if ((*stringVal & 0x80) != 0 || (*stringVal == 0x1b))
                 return OFTrue;
         }
     }