From 7f2e1ed97d84e6dc73bf9a421538415d254af8cf Mon Sep 17 00:00:00 2001 From: Alexey Date: Tue, 13 Dec 2022 22:19:53 +0300 Subject: [PATCH 1/2] Fix bug 60093 --- Test/Applications/x2tTester/README.md | 20 ++++++ Test/Applications/x2tTester/x2tTester.cpp | 78 +++++++++++++++++++++-- Test/Applications/x2tTester/x2tTester.h | 10 +++ 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/Test/Applications/x2tTester/README.md b/Test/Applications/x2tTester/README.md index b5c182bfa0..479c4ec91c 100644 --- a/Test/Applications/x2tTester/README.md +++ b/Test/Applications/x2tTester/README.md @@ -30,6 +30,26 @@ You need to create an xml configuration file. It must contain: # (non-required) timestamp in report file name (default - 1) + # (non-required) is take params for csv/txt conversions from filename (default - 1). + + + # params in any order, in any place + # input file with csv: + filename[cp(codepage)][del%(unicode code)] + # example: + some_csv_file[cpUTF-8][del%3b].csv + + # same with txt file, but only encoding: + filename[cp(codepage)] + # example + some_txt_file[cpUTF-8].txt + + # (non-required) default encoding for csv/txt files, if filenameCsvTxtParams is 0, or no param in filename (default - UTF-8) + + + # (non-required) default unicode code delimiter for csv files, if filenameCsvTxtParams is 0, or no param in filename (default - 3b) + + # (non-required) path to xml file with a list of input files diff --git a/Test/Applications/x2tTester/x2tTester.cpp b/Test/Applications/x2tTester/x2tTester.cpp index 5747800ce3..7c60a117dc 100644 --- a/Test/Applications/x2tTester/x2tTester.cpp +++ b/Test/Applications/x2tTester/x2tTester.cpp @@ -210,6 +210,9 @@ Cx2tTester::Cx2tTester(const std::wstring& configPath) m_bIsErrorsOnly = false; m_bIsTimestamp = true; m_bIsDeleteOk = false; + m_bIsfilenameCsvTxtParams = true; + m_defaultCsvDelimiter = L";"; + m_defaultCsvTxtEndcoding = L"UTF-8"; m_inputFormatsList.SetDefault(); m_outputFormatsList.SetOutput(); SetConfig(configPath); @@ -277,10 +280,13 @@ void Cx2tTester::SetConfig(const std::wstring& configPath) else if(name == L"outputDirectory") m_outputDirectory = node.GetText(); else if(name == L"x2tPath") m_x2tPath = node.GetText(); else if(name == L"cores") m_maxProc = std::stoi(node.GetText()); - else if(name == L"errorsOnly") m_bIsErrorsOnly = std::stoi(node.GetText()); - else if(name == L"timestamp") m_bIsTimestamp = std::stoi(node.GetText()); - else if(name == L"deleteOk") m_bIsDeleteOk = std::stoi(node.GetText()); - else if(name == L"inputFilesList") + else if(name == L"errorsOnly" && !node.GetText().empty()) m_bIsErrorsOnly = std::stoi(node.GetText()); + else if(name == L"timestamp" && !node.GetText().empty()) m_bIsTimestamp = std::stoi(node.GetText()); + else if(name == L"deleteOk" && !node.GetText().empty()) m_bIsDeleteOk = std::stoi(node.GetText()); + else if(name == L"filenameCsvTxtParams" && !node.GetText().empty()) m_bIsfilenameCsvTxtParams = std::stoi(node.GetText()); + else if(name == L"defaultCsvTxtEncoding" && !node.GetText().empty()) m_defaultCsvTxtEndcoding = node.GetText(); + else if(name == L"defaultCsvDelimiter" && !node.GetText().empty()) m_defaultCsvDelimiter = (wchar_t)std::stoi(node.GetText(), nullptr, 16); + else if(name == L"inputFilesList" && !node.GetText().empty()) { XmlUtils::CXmlNode files_list_root; XmlUtils::CXmlNodes files_list_nodes; @@ -300,14 +306,14 @@ void Cx2tTester::SetConfig(const std::wstring& configPath) std::cerr << "Input files list is not open!" << std::endl; } } - else if(name == L"input") + else if(name == L"input" && !node.GetText().empty()) { default_input_formats = false; std::wstring extensions = node.GetText(); extensions += L' '; m_inputFormats = ParseExtensionsString(extensions, m_inputFormatsList); } - else if(name == L"output") + else if(name == L"output" && !node.GetText().empty()) { default_output_formats = false; std::wstring extensions = node.GetText(); @@ -405,6 +411,7 @@ void Cx2tTester::Start() for(int i = 0; i < files.size(); i++) { std::wstring& input_file = files[i]; + std::wstring input_filename = NSFile::GetFileName(input_file); std::wstring input_ext = L'.' + NSFile::GetFileExtention(input_file); int input_format = COfficeFileFormatChecker::GetFormatByExtension(input_ext); @@ -449,6 +456,38 @@ void Cx2tTester::Start() } } + std::wstring csvTxtEncodingS = m_defaultCsvTxtEndcoding; + std::wstring csvDelimiter = m_defaultCsvDelimiter; + + // setup csv & txt additional params + if(m_bIsfilenameCsvTxtParams + || input_format == AVS_OFFICESTUDIO_FILE_DOCUMENT_TXT + || input_format == AVS_OFFICESTUDIO_FILE_SPREADSHEET_CSV) + { + std::wstring find_str = L"[cp"; + size_t pos1 = input_filename.find(find_str); + size_t pos2 = input_filename.find(L"]", pos1 + 1); + if(pos1 != std::wstring::npos && pos2 != std::wstring::npos) + csvTxtEncodingS = input_filename.substr(pos1 + find_str.size(), pos2 - pos1 - find_str.size()); + } + + int csvTxtEncoding; + for(auto &val : NSUnicodeConverter::Encodings) + if(val.Name == U_TO_UTF8(csvTxtEncodingS)) + { + csvTxtEncoding = val.Index; + break; + } + + if(m_bIsfilenameCsvTxtParams || input_format == AVS_OFFICESTUDIO_FILE_SPREADSHEET_CSV) + { + std::wstring find_str = L"[del%"; + size_t pos1 = input_filename.find(find_str); + size_t pos2 = input_filename.find(L"]", pos1 + 1); + if(pos1 != std::wstring::npos && pos2 != std::wstring::npos) + csvDelimiter = (wchar_t)std::stoi(input_filename.substr(pos1 + find_str.size(), pos2 - pos1 - find_str.size()), nullptr, 16); + } + // setup folder for output files if(NSDirectory::Exists(output_files_directory)) NSDirectory::DeleteDirectory(output_files_directory); @@ -473,6 +512,8 @@ void Cx2tTester::Start() converter->SetErrorsOnly(m_bIsErrorsOnly); converter->SetDeleteOk(m_bIsDeleteOk); converter->SetXmlErrorsDirectory(m_errorsXmlDirectory); + converter->SetCsvTxtEncoding(csvTxtEncoding); + converter->SetCsvDelimiter(csvDelimiter); converter->SetFilesCount(files.size(), i + 1); converter->DestroyOnFinish(); m_currentProc++; @@ -617,6 +658,14 @@ void CConverter::SetXmlErrorsDirectory(const std::wstring& errorsXmlDirectory) { m_errorsXmlDirectory = errorsXmlDirectory; } +void CConverter::SetCsvTxtEncoding(int csvTxtEncoding) +{ + m_csvTxtEncoding = csvTxtEncoding; +} +void CConverter::SetCsvDelimiter(std::wstring csvDelimiter) +{ + m_csvDelimiter = csvDelimiter; +} void CConverter::SetFilesCount(int totalFiles, int currFile) { m_totalFiles = totalFiles; @@ -692,6 +741,23 @@ DWORD CConverter::ThreadProc() builder.WriteString(L"2falsefalse10001000"); } + // csv & txt needs encoding param + if(m_inputFormat == AVS_OFFICESTUDIO_FILE_DOCUMENT_TXT || m_inputFormat == AVS_OFFICESTUDIO_FILE_SPREADSHEET_CSV) + { + builder.WriteString(L""); + builder.WriteEncodeXmlString(std::to_wstring(m_csvTxtEncoding)); + builder.WriteString(L""); + } + + // csv needs delimiter param + if(m_inputFormat == AVS_OFFICESTUDIO_FILE_SPREADSHEET_CSV) + { + builder.WriteString(L""); + builder.WriteEncodeXmlString(m_csvDelimiter); + builder.WriteString(L""); + } + + builder.WriteString(L"{"spreadsheetLayout":{"gridLines":true,"headings":true,"fitToHeight":1,"fitToWidth":1,"orientation":"landscape"}}"); builder.WriteString(L""); diff --git a/Test/Applications/x2tTester/x2tTester.h b/Test/Applications/x2tTester/x2tTester.h index 30f50d9798..b357b9104e 100644 --- a/Test/Applications/x2tTester/x2tTester.h +++ b/Test/Applications/x2tTester/x2tTester.h @@ -19,6 +19,7 @@ #include "../../../DesktopEditor/fontengine/ApplicationFontsWorker.h" #include "../../../OfficeUtils/src/OfficeUtils.h" +#include "../../../UnicodeConverter/UnicodeConverter_Encodings.h" class CFormatsList { @@ -125,6 +126,10 @@ private: bool m_bIsErrorsOnly; bool m_bIsTimestamp; bool m_bIsDeleteOk; + bool m_bIsfilenameCsvTxtParams; + + std::wstring m_defaultCsvTxtEndcoding; + std::wstring m_defaultCsvDelimiter; DWORD m_timeStart; }; @@ -145,6 +150,8 @@ public: void SetErrorsOnly(bool bIsErrorsOnly); void SetDeleteOk(bool bIsDeleteOk); void SetXmlErrorsDirectory(const std::wstring& errorsXmlDirectory); + void SetCsvTxtEncoding(int csvTxtEncoding); + void SetCsvDelimiter(std::wstring csvDelimiter); void SetFilesCount(int totalFiles, int currFile); virtual DWORD ThreadProc(); @@ -164,6 +171,9 @@ private: std::wstring m_x2tPath; std::wstring m_errorsXmlDirectory; + int m_csvTxtEncoding; + std::wstring m_csvDelimiter; + bool m_bIsErrorsOnly; bool m_bIsDeleteOk; From 7b1715cfcff6cfea33ec10df8b2e31007c1a27d5 Mon Sep 17 00:00:00 2001 From: Alexey Date: Wed, 14 Dec 2022 14:31:43 +0300 Subject: [PATCH 2/2] Update readme --- Test/Applications/x2tTester/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Test/Applications/x2tTester/README.md b/Test/Applications/x2tTester/README.md index 479c4ec91c..face07efcc 100644 --- a/Test/Applications/x2tTester/README.md +++ b/Test/Applications/x2tTester/README.md @@ -33,7 +33,8 @@ You need to create an xml configuration file. It must contain: # (non-required) is take params for csv/txt conversions from filename (default - 1). - # params in any order, in any place + # params in any order, in any place, delimiter is unicode code in hex + # encoding is case sensitive # input file with csv: filename[cp(codepage)][del%(unicode code)] # example: