From 94f16a89613e9497234c73969319c66b642df235 Mon Sep 17 00:00:00 2001 From: abv Date: Sun, 25 Oct 2020 22:10:27 +0300 Subject: [PATCH] 0014673: Provide true support for Unicode symbols Construction of TCollection_ExtendedString from plain C string is fixed to consider input string as UTF-8 in several places (identified as described in notes to #31113). Message_MsgFile is corrected to load resource file as UTF-8 (unless it has BOM indicating use of UTF-16). Added tests for use of Unicode in some DRAW commands (bugs demo bug14673_*) --- dox/upgrade/upgrade.md | 7 ++++ src/DDataStd/DDataStd_BasicCommands.cxx | 44 ++++++++++++-------- src/DDocStd/DDocStd_ApplicationCommands.cxx | 8 ++-- src/DDocStd/DDocStd_MTMCommands.cxx | 2 +- src/DNaming/DNaming_BasicCommands.cxx | 2 +- src/DNaming/DNaming_ModelingCommands.cxx | 2 +- src/Message/Message_MsgFile.cxx | 16 ++++--- src/TObj/TObj_Assistant.cxx | 2 +- src/TObjDRAW/TObjDRAW.cxx | 9 ++-- src/ViewerTest/ViewerTest_ViewerCommands.cxx | 7 ++-- src/XDEDRAW/XDEDRAW.cxx | 2 +- src/XDEDRAW/XDEDRAW_Notes.cxx | 14 +++---- src/XSDRAWSTLVRML/XSDRAWSTLVRML.cxx | 4 +- tests/bugs/demo/bug14673_1 | 12 ++++++ tests/bugs/demo/bug14673_2 | 44 ++++++++++++++++++++ tests/bugs/demo/bug14673_3 | 22 ++++++++++ tests/bugs/demo/bug14673_4 | 28 +++++++++++++ 17 files changed, 177 insertions(+), 48 deletions(-) create mode 100644 tests/bugs/demo/bug14673_1 create mode 100644 tests/bugs/demo/bug14673_2 create mode 100644 tests/bugs/demo/bug14673_3 create mode 100644 tests/bugs/demo/bug14673_4 diff --git a/dox/upgrade/upgrade.md b/dox/upgrade/upgrade.md index dea0a5fb71..fdb3a8ee03 100644 --- a/dox/upgrade/upgrade.md +++ b/dox/upgrade/upgrade.md @@ -2149,3 +2149,10 @@ For an example, access to labels and attributes could be protected by mutex if t Draw Harness hotkeys **W** (Wireframe) and **S** (Shaded) have been re-mapped to **Ctrl+W** and **Ctrl+S**. Hotkey **A** has been remapped to **Backspace**. Hotkeys WASD and Arrays are now mapped for walk-through navigation in 3D Viewer. + +@subsection upgrade_750_msgfile_utf8 Utf-8 encoding for message files + +Message files (with extension .msg) are now expected to be in UTF-8 encoding (unless they have UTF-16 BOM in which case UTF-16 is expected). +This allows using arbitrary Unicode symbols for localization of messages. + +Existing message files containing 8-bit characters (previously interpreted as characters from Latin-1 code block) should be converted to UTF-8. diff --git a/src/DDataStd/DDataStd_BasicCommands.cxx b/src/DDataStd/DDataStd_BasicCommands.cxx index 6816f6a587..0d7b16d664 100644 --- a/src/DDataStd/DDataStd_BasicCommands.cxx +++ b/src/DDataStd/DDataStd_BasicCommands.cxx @@ -1042,7 +1042,7 @@ static Standard_Integer DDataStd_SetRelation (Draw_Interpretor& di, Standard_CString expr (arg[3]); Handle(TDataStd_Relation) aR = TDataStd_Relation::Set(label); - aR->SetRelation(expr); + aR->SetRelation(TCollection_ExtendedString (expr, Standard_True)); Handle(TDataStd_Variable) aV; for (Standard_Integer i = 4; i < nb; i++) @@ -1222,7 +1222,8 @@ static Standard_Integer DDataStd_SetExtStringArray (Draw_Interpretor& di, if ((!isGuid && nb > 6) || (isGuid && nb > 8)) { j = j + 2; for(Standard_Integer i = From; i<=To; ++i) { - A->SetValue(i, arg[j] ); + TCollection_ExtendedString aVal (arg[j], Standard_True); + A->SetValue(i, aVal); j++; } } @@ -1256,7 +1257,8 @@ static Standard_Integer DDataStd_SetExtStringArrayValue (Draw_Interpretor&, Handle(TDataStd_ExtStringArray) arr; if (label.FindAttribute(TDataStd_ExtStringArray::GetID(), arr)) { - arr->SetValue(index, arg[4]); + TCollection_ExtendedString aVal(arg[4], Standard_True); + arr->SetValue(index, aVal); return 0; } @@ -3444,13 +3446,14 @@ static Standard_Integer DDataStd_GetNDInteger (Draw_Interpretor& di, std::cout <AddComment(D,comment); D->AddComment(comment); diff --git a/src/DDocStd/DDocStd_MTMCommands.cxx b/src/DDocStd/DDocStd_MTMCommands.cxx index 03b56a0ec8..50109074c3 100644 --- a/src/DDocStd/DDocStd_MTMCommands.cxx +++ b/src/DDocStd/DDocStd_MTMCommands.cxx @@ -109,7 +109,7 @@ static int mtmCommitTransaction (Draw_Interpretor& di, int n, const char** a) return 1; } if(n > 1) - sMultiTransactionManager->CommitCommand(a[1]); + sMultiTransactionManager->CommitCommand(TCollection_ExtendedString (a[1], Standard_True)); else sMultiTransactionManager->CommitCommand(); return 0; diff --git a/src/DNaming/DNaming_BasicCommands.cxx b/src/DNaming/DNaming_BasicCommands.cxx index 1ea4b8adee..4a2ca3107b 100644 --- a/src/DNaming/DNaming_BasicCommands.cxx +++ b/src/DNaming/DNaming_BasicCommands.cxx @@ -453,7 +453,7 @@ static Standard_Integer DNaming_ImportShape (Draw_Interpretor& di, const TopoDS_Shape& aShape = DBRep::Get(a[3]); if(aShape.IsNull()) return 1; if(nb == 5) { - TDataStd_Name::Set(L, a[4]); + TDataStd_Name::Set(L, TCollection_ExtendedString (a[4], Standard_True)); } DNaming::LoadImportedShape(L, aShape); diff --git a/src/DNaming/DNaming_ModelingCommands.cxx b/src/DNaming/DNaming_ModelingCommands.cxx index 43791f6d20..bf2bc4b357 100644 --- a/src/DNaming/DNaming_ModelingCommands.cxx +++ b/src/DNaming/DNaming_ModelingCommands.cxx @@ -105,7 +105,7 @@ static Standard_Integer DNaming_AddObject(Draw_Interpretor& di, Handle(TDataStd_UAttribute) anObj = AddObject (aDoc); if(!anObj.IsNull()) { if(nb == 3) - TDataStd_Name::Set(anObj->Label(), a[2]); + TDataStd_Name::Set(anObj->Label(), TCollection_ExtendedString (a[2], Standard_True)); DDF::ReturnLabel(di, anObj->Label()); return 0; } diff --git a/src/Message/Message_MsgFile.cxx b/src/Message/Message_MsgFile.cxx index 22ffbbc56f..31fc76bad7 100644 --- a/src/Message/Message_MsgFile.cxx +++ b/src/Message/Message_MsgFile.cxx @@ -82,13 +82,17 @@ Standard_Boolean Message_MsgFile::Load (const Standard_CString theDirName, //Called : from loadFile() //======================================================================= -template static inline Standard_Boolean -getString (_Char *& thePtr, +template struct TCollection_String; +template <> struct TCollection_String { typedef TCollection_AsciiString type; }; +template <> struct TCollection_String { typedef TCollection_ExtendedString type; }; + +template static inline Standard_Boolean +getString (CharType *& thePtr, TCollection_ExtendedString& theString, Standard_Integer& theLeftSpaces) { - _Char * anEndPtr = thePtr; - _Char * aPtr; + CharType * anEndPtr = thePtr; + CharType * aPtr; Standard_Integer aLeftSpaces; do @@ -98,7 +102,7 @@ getString (_Char *& thePtr, aLeftSpaces = 0; for (;;) { - _Char aChar = * aPtr; + CharType aChar = * aPtr; if (aChar == ' ') aLeftSpaces++; else if (aChar == '\t') aLeftSpaces += 8; else if (aChar == '\r' || * aPtr == '\n') aLeftSpaces = 0; @@ -121,7 +125,7 @@ getString (_Char *& thePtr, thePtr = anEndPtr; if (*thePtr) *thePtr++ = '\0'; - theString = TCollection_ExtendedString (aPtr); + theString = typename TCollection_String::type (aPtr); theLeftSpaces = aLeftSpaces; return Standard_True; } diff --git a/src/TObj/TObj_Assistant.cxx b/src/TObj/TObj_Assistant.cxx index 8ce8e6cbfd..4783d45a46 100644 --- a/src/TObj/TObj_Assistant.cxx +++ b/src/TObj/TObj_Assistant.cxx @@ -76,7 +76,7 @@ Standard_Integer& TObj_Assistant::getVersion() Handle(TObj_Model) TObj_Assistant::FindModel (const Standard_CString theName) { - TCollection_ExtendedString aName( theName ); + TCollection_ExtendedString aName(theName, Standard_True); Standard_Integer i = getModels().Length(); Handle(TObj_Model) aModel; for(; i > 0; i --) diff --git a/src/TObjDRAW/TObjDRAW.cxx b/src/TObjDRAW/TObjDRAW.cxx index da25d2bc97..12b4e3fab7 100644 --- a/src/TObjDRAW/TObjDRAW.cxx +++ b/src/TObjDRAW/TObjDRAW.cxx @@ -194,7 +194,7 @@ static Standard_Integer saveModel (Draw_Interpretor& di, Standard_Integer argc, if ( aModel.IsNull() ) return 1; Standard_Boolean isSaved = Standard_False; if (argc > 2 ) - isSaved = aModel->SaveAs( argv[2] ); + isSaved = aModel->SaveAs( TCollection_ExtendedString (argv[2], Standard_True) ); else isSaved = aModel->Save(); @@ -215,11 +215,12 @@ static Standard_Integer loadModel (Draw_Interpretor& di, Standard_Integer argc, Standard_Boolean isLoaded = Standard_False; Handle(TObj_Model) aModel = getModelByName(argv[1]); + TCollection_ExtendedString aPath(argv[2], Standard_True); if ( aModel.IsNull() ) { // create new aModel = new TObjDRAW_Model(); - isLoaded = aModel->Load( argv[2] ); + isLoaded = aModel->Load(aPath); if ( isLoaded ) { Handle(TDocStd_Document) D = aModel->GetDocument(); @@ -230,7 +231,9 @@ static Standard_Integer loadModel (Draw_Interpretor& di, Standard_Integer argc, } } else - isLoaded = aModel->Load( argv[2] ); + { + isLoaded = aModel->Load(aPath); + } if (!isLoaded) { diff --git a/src/ViewerTest/ViewerTest_ViewerCommands.cxx b/src/ViewerTest/ViewerTest_ViewerCommands.cxx index ffac6b827a..42486337d8 100644 --- a/src/ViewerTest/ViewerTest_ViewerCommands.cxx +++ b/src/ViewerTest/ViewerTest_ViewerCommands.cxx @@ -4690,7 +4690,7 @@ static int VColorScale (Draw_Interpretor& theDI, return 1; } - TCollection_ExtendedString aText (theArgVec[anArgIter + 2]); + TCollection_ExtendedString aText (theArgVec[anArgIter + 2], Standard_True); aColorScale->SetLabel (aText, anIndex); aColorScale->SetLabelType (Aspect_TOCSD_USER); anArgIter += 2; @@ -4797,7 +4797,7 @@ static int VColorScale (Draw_Interpretor& theDI, TColStd_SequenceOfExtendedString aSeq; for (Standard_Integer aLabelIter = 0; aLabelIter < aNbLabels; ++aLabelIter) { - aSeq.Append (TCollection_ExtendedString (theArgVec[++anArgIter])); + aSeq.Append (TCollection_ExtendedString (theArgVec[++anArgIter], Standard_True)); } aColorScale->SetLabels (aSeq); aColorScale->SetLabelType (Aspect_TOCSD_USER); @@ -4839,7 +4839,8 @@ static int VColorScale (Draw_Interpretor& theDI, Standard_ENABLE_DEPRECATION_WARNINGS } - aColorScale->SetTitle (theArgVec[anArgIter + 1]); + TCollection_ExtendedString aTitle(theArgVec[anArgIter + 1], Standard_True); + aColorScale->SetTitle (aTitle); if (isTwoArgs) { anArgIter += 1; diff --git a/src/XDEDRAW/XDEDRAW.cxx b/src/XDEDRAW/XDEDRAW.cxx index 617c3688bc..f737cb3e02 100644 --- a/src/XDEDRAW/XDEDRAW.cxx +++ b/src/XDEDRAW/XDEDRAW.cxx @@ -212,7 +212,7 @@ static Standard_Integer openDoc (Draw_Interpretor& di, Standard_Integer argc, co return 1; } - Standard_CString Filename = argv[1]; + TCollection_AsciiString Filename = argv[1]; Standard_CString DocName = argv[2]; if ( DDocStd::GetDocument(DocName, D, Standard_False) ) diff --git a/src/XDEDRAW/XDEDRAW_Notes.cxx b/src/XDEDRAW/XDEDRAW_Notes.cxx index 42a90f4d89..21ee24349e 100644 --- a/src/XDEDRAW/XDEDRAW_Notes.cxx +++ b/src/XDEDRAW/XDEDRAW_Notes.cxx @@ -215,7 +215,7 @@ noteCreateBalloon(Draw_Interpretor& di, Standard_Integer argc, const char** argv di << "Error: user name is expected.\n" << myCommand; return 1; } - aUsername = argv[iarg]; + aUsername = TCollection_ExtendedString (argv[iarg], Standard_True); } else if (opt == "--time") { @@ -224,7 +224,7 @@ noteCreateBalloon(Draw_Interpretor& di, Standard_Integer argc, const char** argv di << "Error: timestamp is expected.\n" << myCommand; return 1; } - aTimestamp = argv[iarg]; + aTimestamp = TCollection_ExtendedString (argv[iarg], Standard_True); } } @@ -280,7 +280,7 @@ noteCreateComment(Draw_Interpretor& di, Standard_Integer argc, const char** argv di << "Error: user name is expected.\n" << myCommand; return 1; } - aUsername = argv[iarg]; + aUsername = TCollection_ExtendedString (argv[iarg], Standard_True); } else if (opt == "--time") { @@ -289,7 +289,7 @@ noteCreateComment(Draw_Interpretor& di, Standard_Integer argc, const char** argv di << "Error: timestamp is expected.\n" << myCommand; return 1; } - aTimestamp = argv[iarg]; + aTimestamp = TCollection_ExtendedString (argv[iarg], Standard_True); } } @@ -355,7 +355,7 @@ noteCreateBinData(Draw_Interpretor& di, Standard_Integer argc, const char** argv di << "Error: file path is expected.\n" << myCommand; return 1; } - aFilename = argv[iarg]; + aFilename = TCollection_ExtendedString (argv[iarg], Standard_True); aFromFile = Standard_True; } else if (opt == "--data") @@ -396,7 +396,7 @@ noteCreateBinData(Draw_Interpretor& di, Standard_Integer argc, const char** argv di << "Error: user name is expected.\n" << myCommand; return 1; } - aUsername = argv[iarg]; + aUsername = TCollection_ExtendedString (argv[iarg], Standard_True); } else if (opt == "--time") { @@ -405,7 +405,7 @@ noteCreateBinData(Draw_Interpretor& di, Standard_Integer argc, const char** argv di << "Error: timestamp is expected.\n" << myCommand; return 1; } - aTimestamp = argv[iarg]; + aTimestamp = TCollection_ExtendedString (argv[iarg], Standard_True); } } diff --git a/src/XSDRAWSTLVRML/XSDRAWSTLVRML.cxx b/src/XSDRAWSTLVRML/XSDRAWSTLVRML.cxx index 0eb954617b..3d8b497b4e 100644 --- a/src/XSDRAWSTLVRML/XSDRAWSTLVRML.cxx +++ b/src/XSDRAWSTLVRML/XSDRAWSTLVRML.cxx @@ -201,7 +201,7 @@ static Standard_Integer ReadGltf (Draw_Interpretor& theDI, else { Handle(DDocStd_DrawDocument) aDrawDoc = new DDocStd_DrawDocument (aDoc); - TDataStd_Name::Set (aDoc->GetData()->Root(), aDestName.ToCString()); + TDataStd_Name::Set (aDoc->GetData()->Root(), aDestName); Draw::Set (aDestName.ToCString(), aDrawDoc); } } @@ -604,7 +604,7 @@ static Standard_Integer ReadObj (Draw_Interpretor& theDI, else { Handle(DDocStd_DrawDocument) aDrawDoc = new DDocStd_DrawDocument (aDoc); - TDataStd_Name::Set (aDoc->GetData()->Root(), aDestName.ToCString()); + TDataStd_Name::Set (aDoc->GetData()->Root(), aDestName); Draw::Set (aDestName.ToCString(), aDrawDoc); } } diff --git a/tests/bugs/demo/bug14673_1 b/tests/bugs/demo/bug14673_1 new file mode 100644 index 0000000000..8a437c7ef1 --- /dev/null +++ b/tests/bugs/demo/bug14673_1 @@ -0,0 +1,12 @@ +puts "# ============================================================" +puts "# 0014673: Provide true support for Unicode symbols" +puts "# ============================================================" +puts "" +puts "# Use non-Ascii names for color scale" + +pload VISUALIZATION +vinit +vcolorscale languages -title "Языки Восточной Азии" -range 1 3 3 -colors red green blue \ + -labelAtBorder off -labels "Japanese 日本語" "Korean 한국어" "Chinese 中文" + +vdump ${imagedir}/${casename}.png diff --git a/tests/bugs/demo/bug14673_2 b/tests/bugs/demo/bug14673_2 new file mode 100644 index 0000000000..ec4167204d --- /dev/null +++ b/tests/bugs/demo/bug14673_2 @@ -0,0 +1,44 @@ +puts "# ============================================================" +puts "# 0014673: Provide true support for Unicode symbols" +puts "# ============================================================" +puts "" +puts "# Check that different file open / save commands can deal with Unicode" + +pload ALL + +# Name means "Japanese" +set filePrefix "$imagedir/${casename}_日本語" + +proc checkFile {filepath} { + if { [file exists $filepath] } { + puts "File $filepath is ceated, OK" + } else { + puts "Error: Could not find file $filepath" + } +} + +puts "# OCAF documents" +NewDocument Xml XmlOcaf +SaveAs Xml ${filePrefix}.xml +checkFile ${filePrefix}.xml +Close Xml +Open ${filePrefix}.xml Xml + +NewDocument Bin BinOcaf +SaveAs Bin ${filePrefix}.cbf +checkFile ${filePrefix}.cbf +Close Bin +Open ${filePrefix}.cbf Bin + +puts "# STEP and IGES" +box b 10 10 10 +NewDocument XDE BinOcaf +XAddShape XDE b + +WriteStep XDE ${filePrefix}.stp +checkFile ${filePrefix}.stp +ReadStep Step ${filePrefix}.stp + +WriteIges XDE ${filePrefix}.igs +checkFile ${filePrefix}.igs +ReadIges Iges ${filePrefix}.igs diff --git a/tests/bugs/demo/bug14673_3 b/tests/bugs/demo/bug14673_3 new file mode 100644 index 0000000000..03f4e2d750 --- /dev/null +++ b/tests/bugs/demo/bug14673_3 @@ -0,0 +1,22 @@ +puts "# ============================================================" +puts "# 0014673: Provide true support for Unicode symbols" +puts "# ============================================================" +puts "" +puts "# Check that resource file with messages saved in file with UTF-8" +puts "# encoding is correctly loaded and processed" + +puts "# Prepare resource file to be used instead of standard XSTEP.us," +puts "# with Russian translation of one message" +dsetenv CSF_LANGUAGE ru +dsetenv CSF_XSMessage $imagedir +set message "Б-сплайн поверхность не является гладкой" +set fd [open $imagedir/XSTEP.ru w] +fconfigure $fd -encoding utf-8 +puts $fd ".XSTEP_1\nMessage used to check file load\n.IGES_1250\n$message" +close $fd + +puts "# Load IGES file which is known to generate the message, and check it" +puts "REQUIRED 14673 ALL: $message" +pload XSDRAW +igesbrep [locate_data_file hammer.iges] a * +tpstat c diff --git a/tests/bugs/demo/bug14673_4 b/tests/bugs/demo/bug14673_4 new file mode 100644 index 0000000000..54987074f4 --- /dev/null +++ b/tests/bugs/demo/bug14673_4 @@ -0,0 +1,28 @@ +puts "# ============================================================" +puts "# 0014673: Provide true support for Unicode symbols" +puts "# ============================================================" +puts "" +puts "# Check that non-Ascii text strings can be saved and restored in OCAF" + +puts "# Prepare OCAF document with text strings in different languages" +set strings [list "test" "l'épreuve" "опыт" "테스트" "größten 市"] + +pload OCAF +NewDocument D XmlOcaf +set i 0 +foreach str $strings { + SetName D 0:[incr i] $str +} + +puts "# Save it and load back, then check the strings" +SaveAs D $imagedir/${casename}.xml +Close D +Open $imagedir/${casename}.xml D + +set i 0 +foreach str $strings { + set res [GetName D 0:[incr i]] + if { $res != $str } { + puts "Error: string $str was restored as $res" + } +} -- 2.20.1