Fix Alpha type SoftMask and nesting of layers

DesktopEditor/agg-2.4/include/agg_alpha_mask_u8.h

@@ -30,6 +30,12 @@ namespace agg
     {
         static unsigned calculate(const int8u* p) { return *p; }
     };
+
+	//===================================================four_component_mask_u8
+	struct four_component_mask_u8
+	{
+		static unsigned calculate(const int8u* p) { return p[3]; }
+	};
     
 
     //=====================================================rgb_to_gray_mask_u8

DesktopEditor/graphics/AlphaMask.cpp

@@ -33,9 +33,15 @@ namespace Aggplus
 		{
 		case EMaskDataType::ImageBuffer: return 4;
 		case EMaskDataType::AlphaBuffer: return 1;
+		case EMaskDataType::Alpha4Buffer: return 4;
 		}
 	}
 
+	void CAlphaMask::SetDataType(EMaskDataType oType)
+	{
+		m_enDataType = oType;
+	}
+
 	Status CAlphaMask::Create(UINT unWidth, UINT unHeight, EMaskDataType eDataType)
 	{
 		if (0 == unWidth || 0 == unHeight)

DesktopEditor/graphics/AlphaMask.h

@@ -11,7 +11,8 @@ namespace Aggplus
 	enum class EMaskDataType
 	{
 		ImageBuffer,
-		AlphaBuffer
+		AlphaBuffer,
+		Alpha4Buffer
 	};
 
 	class GRAPHICS_DECL CAlphaMask : public IGrObject
@@ -25,6 +26,7 @@ namespace Aggplus
 		EMaskDataType GetDataType() const;
 		UINT          GetStep() const;
 
+		void SetDataType(EMaskDataType oType);
 		Status Create(UINT unWidth, UINT unHeight, EMaskDataType eDataType);
 		Status LoadFromBuffer(BYTE* pBuffer, EMaskDataType eDataType, bool bExternalBuffer = true);
 	private:

DesktopEditor/graphics/Graphics.cpp

@@ -1236,7 +1236,9 @@ namespace Aggplus
 		if (m_pAlphaMask)
 			m_pAlphaMask->AddRef();
 
-		return CreateLayer();
+		return Ok;
+
+		//return CreateLayer();
 	}
 
 	Status CGraphics::StartCreatingAlphaMask()
@@ -1355,6 +1357,11 @@ namespace Aggplus
 					Aggplus::BlendTo<agg::one_component_mask_u8>(pCurrentGraphicsLayer, m_frame_buffer.pixfmt(), m_pAlphaMask->GetBuffer(), m_pAlphaMask->GetStep());
 					break;
 				}
+				case EMaskDataType::Alpha4Buffer:
+				{
+				Aggplus::BlendTo<agg::four_component_mask_u8>(pCurrentGraphicsLayer, m_frame_buffer.pixfmt(), m_pAlphaMask->GetBuffer(), m_pAlphaMask->GetStep());
+				break;
+				}
 			}
 		}
 
@@ -1436,6 +1443,15 @@ namespace Aggplus
 		return Ok;
 	}
 
+	Status CGraphics::SetAlphaMaskType(EMaskDataType oType)
+	{
+		if (!m_pAlphaMask)
+			return WrongState;
+
+		m_pAlphaMask->SetDataType(oType);
+		return Ok;
+	}
+
 	void CGraphics::CalculateFullTransform()
 	{
 		m_oFullTransform = m_oCoordTransform;
@@ -1447,6 +1463,16 @@ namespace Aggplus
 		return m_oClip.IsClip();
 	}
 
+	unsigned int CGraphics::GetLayerW()
+	{
+		return m_frame_buffer.ren_buf().width();
+	}
+
+	unsigned int CGraphics::GetLayerH()
+	{
+		return m_frame_buffer.ren_buf().height();
+	}
+
 	template<class Renderer>
 	void CGraphics::render_scanlines(Renderer& ren)
 	{

DesktopEditor/graphics/Graphics.h

@@ -402,6 +402,7 @@ public:
 
 	//Работа с альфа-маской
 	Status SetAlphaMask(CAlphaMask* pAlphaMask);
+	inline CAlphaMask* GetAlphaMask() { return m_pAlphaMask; }
 	Status StartCreatingAlphaMask();
 	Status EndCreatingAlphaMask();
 	Status ResetAlphaMask();
@@ -416,6 +417,7 @@ public:
 	Status SetLayerOpacity(double dOpacity);
 	Status SetLayerIsolated(bool bIsolated);
 	Status SetAlphaMaskIsolated(bool bIsolated);
+	Status SetAlphaMaskType(EMaskDataType oType);
 
 	void CalculateFullTransform();
 	bool IsClip();
@@ -425,6 +427,8 @@ public:
 
 	inline double GetPixW() { return m_dWidthPix; }
 	inline double GetPixH() { return m_dHeightPix; }
+	unsigned int GetLayerW();
+	unsigned int GetLayerH();
 
 protected:
 

DesktopEditor/graphics/GraphicsRenderer.cpp

@@ -1416,6 +1416,11 @@ void CGraphicsRenderer::SetAlphaMask(Aggplus::CAlphaMask *pAlphaMask)
 	m_pRenderer->SetAlphaMask(pAlphaMask);
 }
 
+Aggplus::CAlphaMask* CGraphicsRenderer::GetAlphaMask()
+{
+	return m_pRenderer->GetAlphaMask();
+}
+
 HRESULT CGraphicsRenderer::put_LayerOpacity(double dValue)
 {
 	return m_pRenderer->SetLayerOpacity(dValue);
@@ -1431,6 +1436,11 @@ HRESULT CGraphicsRenderer::put_AlphaMaskIsolated(bool bIsolated)
 	return m_pRenderer->SetAlphaMaskIsolated(bIsolated);
 }
 
+void CGraphicsRenderer::put_AlphaMaskType(Aggplus::EMaskDataType oType)
+{
+	m_pRenderer->SetAlphaMaskType(oType);
+}
+
 void CGraphicsRenderer::put_GlobalAlphaEnabled(const bool& bEnabled, const double& dVal)
 {
 	m_bGlobalAlphaEnabled = bEnabled;

DesktopEditor/graphics/GraphicsRenderer.h

@@ -348,14 +348,18 @@ public:
 
 	inline double GetPixW() { return m_pRenderer->GetPixW(); }
 	inline double GetPixH() { return m_pRenderer->GetPixH(); }
+	inline unsigned int GetLayerW() override { return m_pRenderer->GetLayerW(); }
+	inline unsigned int GetLayerH() override { return m_pRenderer->GetLayerH(); }
 
 	// alpha mask methods
 	void SetAlphaMask(Aggplus::CAlphaMask* pAlphaMask);
+	Aggplus::CAlphaMask* GetAlphaMask();
 
 	// layer methods
 	virtual HRESULT put_LayerOpacity(double dValue) override;
 	virtual HRESULT put_LayerIsolated(bool bIsolated) override;
 	virtual HRESULT put_AlphaMaskIsolated(bool bIsolated) override;
+	virtual void put_AlphaMaskType(Aggplus::EMaskDataType oType) override;
 
 	// smart methods
 	void drawHorLine(BYTE align, double y, double x, double r, double penW)

DesktopEditor/graphics/pro/Graphics.h

@@ -117,9 +117,13 @@ namespace NSGraphics
 		virtual void Stroke() = 0;
 		virtual double GetPixW() = 0;
 		virtual double GetPixH() = 0;
+		virtual unsigned int GetLayerW() = 0;
+		virtual unsigned int GetLayerH() = 0;
 
 		//alpha mask methods
 		virtual void SetAlphaMask(Aggplus::CAlphaMask* pAlphaMask) = 0;
+		virtual Aggplus::CAlphaMask* GetAlphaMask() = 0;
+		virtual void put_AlphaMaskType(Aggplus::EMaskDataType oType) = 0;
 
 		// smart methods
 		virtual void drawHorLine(BYTE align, double y, double x, double r, double penW) = 0;

PdfFile/SrcReader/RendererOutputDev.cpp

@@ -52,6 +52,7 @@
 #include "../../DesktopEditor/common/Array.h"
 #include "../../DesktopEditor/graphics/BaseThread.h"
 #include "../../DesktopEditor/graphics/commands/DocInfo.h"
+#include "../../DesktopEditor/graphics/AlphaMask.h"
 #include "../Resources/BaseFonts.h"
 #include <new>
 
@@ -548,7 +549,6 @@ namespace PdfReader
 
 		m_pbBreak = NULL;
 
-		m_pGStateSoftMask = NULL;
 		m_bDrawOnlyText = false;
 		m_bClipChanged = true;
 	}
@@ -584,6 +584,7 @@ namespace PdfReader
 	}
 	void RendererOutputDev::endPage()
 	{
+		m_pRenderer->EndCommand(c_nResetMaskType);
 		m_pRenderer->EndCommand(c_nPageType);
 	}
 	void RendererOutputDev::saveState(GfxState *pGState)
@@ -591,18 +592,38 @@ namespace PdfReader
 		m_sClip.push_back(GfxClip());
 		m_bClipChanged = true;
 		updateAll(pGState);
+
+		Aggplus::CAlphaMask* pAlphaMask = NULL;
+		GfxState* pGStateSoftMask = NULL;
+		if (!m_sStates.empty() && m_sStates.back().pAlphaMask)
+		{
+			pAlphaMask = m_sStates.back().pAlphaMask;
+			pGStateSoftMask = m_sStates.back().pGStateSoftMask;
+		}
+		m_sStates.push_back(GfxOutputState());
+		m_sStates.back().pGState = pGState;
+		m_sStates.back().pAlphaMask = pAlphaMask;
+		m_sStates.back().pGStateSoftMask = pGStateSoftMask;
 	}
 	void RendererOutputDev::restoreState(GfxState *pGState)
 	{
-		if (m_pGStateSoftMask == pGState)
+		NSGraphics::IGraphicsRenderer* GRenderer = dynamic_cast<NSGraphics::IGraphicsRenderer*>(m_pRenderer);
+		if (GRenderer)
 		{
-			m_pRenderer->EndCommand(c_nResetMaskType);
+			if (m_sStates.back().pAlphaMask && !GRenderer->GetAlphaMask())
-			m_pGStateSoftMask = NULL;
+				GRenderer->SetAlphaMask(m_sStates.back().pAlphaMask);
+			if (m_sStates.back().pGStateSoftMask == pGState)
+			{
+				m_pRenderer->EndCommand(c_nResetMaskType);
+				RELEASEINTERFACE(m_sStates.back().pAlphaMask);
+			}
 		}
 		if (!m_sClip.empty())
 			m_sClip.pop_back();
 		m_bClipChanged = true;
 		updateAll(pGState);
+
+		m_sStates.pop_back();
 	}
 	void RendererOutputDev::updateCTM(GfxState *pGState, double dMatrix11, double dMatrix12, double dMatrix21, double dMatrix22, double dMatrix31, double dMatrix32)
 	{
@@ -4739,6 +4760,10 @@ namespace PdfReader
 	}
 	void RendererOutputDev::beginTransparencyGroup(GfxState *pGState, double *pBBox, GfxColorSpace *pBlendingColorSpace, GBool bIsolated, GBool bKnockout, GBool bForSoftMask)
 	{
+		NSGraphics::IGraphicsRenderer* GRenderer = dynamic_cast<NSGraphics::IGraphicsRenderer*>(m_pRenderer);
+		if (!GRenderer)
+			return;
+
 		if (bForSoftMask)
 		{
 			m_pRenderer->BeginCommand(c_nMaskType);
@@ -4746,8 +4771,11 @@ namespace PdfReader
 		}
 		else
 		{
-			m_pRenderer->BeginCommand(c_nLayerType);
+			if (!GRenderer->GetAlphaMask())
-			//m_pRenderer->put_LayerIsolated(bIsolated);
+			{
+				m_pRenderer->BeginCommand(c_nLayerType);
+				//m_pRenderer->put_LayerIsolated(bIsolated);
+			}
 		}
 	}
 	void RendererOutputDev::endTransparencyGroup(GfxState *pGState)
@@ -4755,22 +4783,38 @@ namespace PdfReader
 	}
 	void RendererOutputDev::paintTransparencyGroup(GfxState *pGState, double *pBBox)
 	{
+		NSGraphics::IGraphicsRenderer* GRenderer = dynamic_cast<NSGraphics::IGraphicsRenderer*>(m_pRenderer);
+		if (!GRenderer)
+			return;
+
 		double dOpacity = std::min(1.0, std::max(0.0, pGState->getFillOpacity()));
 		m_pRenderer->put_LayerOpacity(dOpacity);
+		if (m_sStates.back().pAlphaMask && !GRenderer->GetAlphaMask())
+			GRenderer->SetAlphaMask(m_sStates.back().pAlphaMask);
 		m_pRenderer->EndCommand(c_nLayerType);
 	}
 	void RendererOutputDev::setSoftMask(GfxState *pGState, double *pBBox, GBool bAlpha, Function *pTransferFunc, GfxColor *pBackdropColor)
 	{
+		NSGraphics::IGraphicsRenderer* GRenderer = dynamic_cast<NSGraphics::IGraphicsRenderer*>(m_pRenderer);
+		if (!GRenderer)
+			return;
+
 		m_pRenderer->EndCommand(c_nMaskType);
-		m_pGStateSoftMask = pGState;
+		if (bAlpha)
+			GRenderer->put_AlphaMaskType(Aggplus::EMaskDataType::Alpha4Buffer);
+		m_sStates.back().pGStateSoftMask = m_sStates.back().pGState;
+		m_sStates.back().pAlphaMask = GRenderer->GetAlphaMask();
+		m_sStates.back().pAlphaMask->AddRef();
 	}
 	void RendererOutputDev::clearSoftMask(GfxState *pGState)
 	{
-		if (m_pGStateSoftMask)
+		NSGraphics::IGraphicsRenderer* GRenderer = dynamic_cast<NSGraphics::IGraphicsRenderer*>(m_pRenderer);
-		{
+		if (!GRenderer)
-			m_pRenderer->EndCommand(c_nResetMaskType);
+			return;
-			m_pGStateSoftMask = NULL;
+
-		}
+		GRenderer->SetAlphaMask(NULL);
+		m_sStates.back().pAlphaMask = NULL;
+		m_sStates.back().pGStateSoftMask = NULL;
 	}
 	void RendererOutputDev::NewPDF(XRef *pXref)
 	{

PdfFile/SrcReader/RendererOutputDev.h

@@ -292,6 +292,15 @@ namespace PdfReader
 		static void CheckFontStylePDF(std::wstring& sName, bool& bBold, bool& bItalic);
 
 	private:
+		struct GfxOutputState
+		{
+			GfxState* pGState;
+
+			GfxState* pGStateSoftMask;
+			Aggplus::CAlphaMask* pAlphaMask;
+
+			GfxOutputState() : pGState(NULL), pGStateSoftMask(NULL), pAlphaMask(NULL) {}
+		};
 
 		void Transform(double *pMatrix, double dUserX, double dUserY, double *pdDeviceX, double *pdDeviceY);
 		void Distance(double *pMatrix, double dUserX, double dUserY, double *pdDeviceX, double *pdDeviceY);
@@ -312,11 +321,11 @@ namespace PdfReader
 
 		bool                         *m_pbBreak;         // Внешняя остановка рендерера
 
+		std::deque<GfxOutputState>    m_sStates;
 		std::deque<GfxClip>           m_sClip;
 		bool                          m_bClipChanged;
 
 		bool                          m_bTiling;
-		GfxState*                     m_pGStateSoftMask;
 
         bool                          m_bDrawOnlyText; // Special option for html-renderer