From 197525991acd47a8ed04be2f86988ef4e0bd4aa8 Mon Sep 17 00:00:00 2001 From: Oleg Korshul Date: Thu, 16 Jul 2020 20:55:18 +0300 Subject: [PATCH] Fix open xmlfiles with non-utf8 encodings --- DesktopEditor/xml/include/xmlencoding.h | 77 ++++++++++++++++++++++++ DesktopEditor/xml/src/xmllight_private.h | 5 +- Fb2File/Fb2File.cpp | 19 ++++-- Fb2File/Fb2File.pro | 2 +- Fb2File/test/main.cpp | 2 - 5 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 DesktopEditor/xml/include/xmlencoding.h diff --git a/DesktopEditor/xml/include/xmlencoding.h b/DesktopEditor/xml/include/xmlencoding.h new file mode 100644 index 0000000000..eeffc51b56 --- /dev/null +++ b/DesktopEditor/xml/include/xmlencoding.h @@ -0,0 +1,77 @@ +/* + * (c) Copyright Ascensio System SIA 2010-2019 + * + * This program is a free software product. You can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License (AGPL) + * version 3 as published by the Free Software Foundation. In accordance with + * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect + * that Ascensio System SIA expressly excludes the warranty of non-infringement + * of any third-party rights. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For + * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html + * + * You can contact Ascensio System SIA at 20A-12 Ernesta Birznieka-Upisha + * street, Riga, Latvia, EU, LV-1050. + * + * The interactive user interfaces in modified source and object code versions + * of the Program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU AGPL version 3. + * + * Pursuant to Section 7(b) of the License you must retain the original Product + * logo when distributing the program. Pursuant to Section 7(e) we decline to + * grant you any rights under trademark law for use of our trademarks. + * + * All the Product's GUI elements, including illustrations and icon sets, as + * well as technical writing content are licensed under the terms of the + * Creative Commons Attribution-ShareAlike 4.0 International. See the License + * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + * + */ +#ifndef _BUILD_XMLUTILS_ENCODINGS_H_ +#define _BUILD_XMLUTILS_ENCODINGS_H_ + +#include "../../common/File.h" +#include "../../../UnicodeConverter/UnicodeConverter.h" +#define XML_HEADER_CHECKER_LENGHT 100 + +namespace XmlUtils +{ + // эта функция считает, что на вход пришла точно хмл. + // надо лишь определить кодировку и отдать строку в utf-8 + static std::string GetXmlContentAsUTF8(const std::wstring& sFile) + { + std::string sXmlSource; + if (!NSFile::CFileBinary::ReadAllTextUtf8A(sFile, sXmlSource)) + return sXmlSource; + + std::string::size_type nCheckCount = sXmlSource.length(); + std::string sChecker = (nCheckCount > XML_HEADER_CHECKER_LENGHT) ? sXmlSource.substr(0, XML_HEADER_CHECKER_LENGHT) : sXmlSource; + + std::string::size_type posEncoding = sChecker.find("encoding=\""); + if (std::string::npos == posEncoding) + return sXmlSource; + + posEncoding += 10; // len(encoding=\") + std::string::size_type posEnd = sChecker.find("\"", posEncoding); + if (std::string::npos == posEnd) + return sXmlSource; + + std::string sEncoding = sChecker.substr(posEncoding, posEnd - posEncoding); + if (sEncoding == "utf-8" || sEncoding == "UTF-8") + return sXmlSource; + + posEnd = sChecker.find(">", posEnd); + if (std::string::npos == posEnd) + return sXmlSource; + + sXmlSource = "" + sXmlSource.substr(posEnd + 1); + + NSUnicodeConverter::CUnicodeConverter oConverter; + std::wstring sUnicodeContent = oConverter.toUnicode(sXmlSource, sEncoding.c_str()); + return U_TO_UTF8(sUnicodeContent); + } +} + +#endif // _BUILD_XMLUTILS_ENCODINGS_H_ diff --git a/DesktopEditor/xml/src/xmllight_private.h b/DesktopEditor/xml/src/xmllight_private.h index 8a50a89310..23cb7d4f7e 100644 --- a/DesktopEditor/xml/src/xmllight_private.h +++ b/DesktopEditor/xml/src/xmllight_private.h @@ -122,7 +122,10 @@ namespace XmlUtils { Clear(); - reader = xmlReaderForMemory((char*)sXml.c_str(), sXml.length(), NULL, NULL, 0); + m_lStreamLen = (LONG)sXml.length(); + m_pStream = new BYTE[m_lStreamLen]; + memcpy(m_pStream, sXml.c_str(), m_lStreamLen); + reader = xmlReaderForMemory((char*)m_pStream, m_lStreamLen, NULL, NULL, 0); return true; } diff --git a/Fb2File/Fb2File.cpp b/Fb2File/Fb2File.cpp index 47acb4d450..c46eba5201 100644 --- a/Fb2File/Fb2File.cpp +++ b/Fb2File/Fb2File.cpp @@ -1,5 +1,6 @@ #include "Fb2File.h" #include "../DesktopEditor/xml/include/xmlutils.h" +#include "../DesktopEditor/xml/include/xmlencoding.h" #include "../DesktopEditor/common/Base64.h" #include "../DesktopEditor/common/File.h" #include "../DesktopEditor/common/Directory.h" @@ -190,6 +191,14 @@ public: delete m_pPublishInfo; } + // wrapper для чтения из файла + bool OpenFromFile(const std::wstring& sFile) + { + // считаем, что это хмл, так как проверка на формат будет раньше (x2t) + std::string sFileContentUtf8 = XmlUtils::GetXmlContentAsUTF8(sFile); + return m_oLightReader.FromStringA(sFileContentUtf8); + } + // Читает и проверят соответствует ли текущий раздел ожиданиям bool isSection(const std::wstring& sName) { @@ -230,7 +239,7 @@ public: bool getImages(const std::wstring& sPath, const std::wstring& sMediaDirectory, NSStringUtils::CStringBuilder& oRels) { // Открывает файл - if (!m_oLightReader.FromFile(sPath)) + if (!OpenFromFile(sPath)) return false; // Читаем FictionBook if(!isSection(L"FictionBook")) @@ -743,7 +752,7 @@ public: void readNotes(const std::wstring& sPath, NSStringUtils::CStringBuilder& oFootnotes) { // Открываем файл - m_oLightReader.FromFile(sPath); + OpenFromFile(sPath); // Читаем FictionBook isSection(L"FictionBook"); @@ -868,7 +877,7 @@ public: bool readContents(const std::wstring& sPath, NSStringUtils::CStringBuilder& oContents) { // Открывает файл - if (!m_oLightReader.FromFile(sPath)) + if (!OpenFromFile(sPath)) return false; // Читаем FictionBook if(!isSection(L"FictionBook")) @@ -1219,7 +1228,7 @@ CFb2File::~CFb2File() bool CFb2File::IsFb2File(const std::wstring& sFile) { // Открывает файл на проверку - if (!m_internal->m_oLightReader.FromFile(sFile)) + if (!m_internal->OpenFromFile(sFile)) return false; // Читаем FictionBook @@ -1293,7 +1302,7 @@ int CFb2File::Convert(const std::wstring& sPath, const std::wstring& sDirectory, m_internal->readNotes(sPath, oFootnotes); // Открываем файл на чтение - if(!m_internal->m_oLightReader.FromFile(sPath)) + if(!m_internal->OpenFromFile(sPath)) return FALSE; // Читаем FictionBook diff --git a/Fb2File/Fb2File.pro b/Fb2File/Fb2File.pro index d7efb4878c..ccae481b44 100644 --- a/Fb2File/Fb2File.pro +++ b/Fb2File/Fb2File.pro @@ -16,7 +16,7 @@ CORE_ROOT_DIR = $$PWD/../../core PWD_ROOT_DIR = $$PWD include($$CORE_ROOT_DIR/Common/base.pri) -ADD_DEPENDENCY(kernel, graphics) +ADD_DEPENDENCY(kernel, UnicodeConverter, graphics) SOURCES += Fb2File.cpp diff --git a/Fb2File/test/main.cpp b/Fb2File/test/main.cpp index 0034c40052..e272de8307 100644 --- a/Fb2File/test/main.cpp +++ b/Fb2File/test/main.cpp @@ -14,8 +14,6 @@ void getDirectories(std::wstring sDirectory, std::vector& arrDirec int main() { - setlocale(LC_ALL, "Russian"); - bool bBatchMode = true; if(bBatchMode) {