# HG changeset patch # User Lee Salzman # Date 1527103806 14400 # Wed May 23 15:30:06 2018 -0400 # Node ID 29fcadf4eecb9569b7029c169a129235f1beeec3 # Parent 5e72df3505f776e74e2ce4141f5fc92b0d488bfe cleanup of swizzle stride calculations MozReview-Commit-ID: GMXRKnu8zHB diff --git a/gfx/2d/DataSurfaceHelpers.cpp b/gfx/2d/DataSurfaceHelpers.cpp --- a/gfx/2d/DataSurfaceHelpers.cpp +++ b/gfx/2d/DataSurfaceHelpers.cpp @@ -152,37 +152,44 @@ UniquePtr SurfaceToPackedBGRA(DataSourceSurface *aSurface) { SurfaceFormat format = aSurface->GetFormat(); if (format != SurfaceFormat::B8G8R8A8 && format != SurfaceFormat::B8G8R8X8) { return nullptr; } IntSize size = aSurface->GetSize(); - - UniquePtr imageBuffer( - new (std::nothrow) uint8_t[size.width * size.height * sizeof(uint32_t)]); + if (size.width < 0 || size.width >= INT32_MAX / 4) { + return nullptr; + } + int32_t stride = size.width * 4; + CheckedInt bufferSize = + CheckedInt(stride) * CheckedInt(size.height); + if (!bufferSize.isValid()) { + return nullptr; + } + UniquePtr imageBuffer(new (std::nothrow) uint8_t[bufferSize.value()]); if (!imageBuffer) { return nullptr; } DataSourceSurface::MappedSurface map; if (!aSurface->Map(DataSourceSurface::MapType::READ, &map)) { return nullptr; } CopySurfaceDataToPackedArray(map.mData, imageBuffer.get(), size, - map.mStride, 4 * sizeof(uint8_t)); + map.mStride, 4); aSurface->Unmap(); if (format == SurfaceFormat::B8G8R8X8) { // Convert BGRX to BGRA by setting a to 255. - SwizzleData(imageBuffer.get(), size.width * sizeof(uint32_t), SurfaceFormat::X8R8G8B8_UINT32, - imageBuffer.get(), size.width * sizeof(uint32_t), SurfaceFormat::A8R8G8B8_UINT32, + SwizzleData(imageBuffer.get(), stride, SurfaceFormat::X8R8G8B8_UINT32, + imageBuffer.get(), stride, SurfaceFormat::A8R8G8B8_UINT32, size); } return imageBuffer; } uint8_t* SurfaceToPackedBGR(DataSourceSurface *aSurface) @@ -191,30 +198,38 @@ SurfaceToPackedBGR(DataSourceSurface *aS MOZ_ASSERT(format == SurfaceFormat::B8G8R8X8, "Format not supported"); if (format != SurfaceFormat::B8G8R8X8) { // To support B8G8R8A8 we'd need to un-pre-multiply alpha return nullptr; } IntSize size = aSurface->GetSize(); - - uint8_t* imageBuffer = new (std::nothrow) uint8_t[size.width * size.height * 3 * sizeof(uint8_t)]; + if (size.width < 0 || size.width >= INT32_MAX / 3) { + return nullptr; + } + int32_t stride = size.width * 3; + CheckedInt bufferSize = + CheckedInt(stride) * CheckedInt(size.height); + if (!bufferSize.isValid()) { + return nullptr; + } + uint8_t* imageBuffer = new (std::nothrow) uint8_t[bufferSize.value()]; if (!imageBuffer) { return nullptr; } DataSourceSurface::MappedSurface map; if (!aSurface->Map(DataSourceSurface::MapType::READ, &map)) { delete [] imageBuffer; return nullptr; } SwizzleData(map.mData, map.mStride, SurfaceFormat::B8G8R8X8, - imageBuffer, size.width * 3, SurfaceFormat::B8G8R8, + imageBuffer, stride, SurfaceFormat::B8G8R8, size); aSurface->Unmap(); return imageBuffer; } void diff --git a/gfx/2d/Swizzle.cpp b/gfx/2d/Swizzle.cpp --- a/gfx/2d/Swizzle.cpp +++ b/gfx/2d/Swizzle.cpp @@ -254,38 +254,52 @@ PremultiplyFallback(const uint8_t* aSrc, // If rows are tightly packed, and the size of the total area will fit within // the precision range of a single row, then process all the data as if it was // a single row. static inline IntSize CollapseSize(const IntSize& aSize, int32_t aSrcStride, int32_t aDstStride) { if (aSrcStride == aDstStride && - aSrcStride == 4 * aSize.width) { + (aSrcStride & 3) == 0 && + aSrcStride / 4 == aSize.width) { CheckedInt32 area = CheckedInt32(aSize.width) * CheckedInt32(aSize.height); if (area.isValid()) { return IntSize(area.value(), 1); } } return aSize; } +static inline int32_t +GetStrideGap(int32_t aWidth, SurfaceFormat aFormat, int32_t aStride) +{ + CheckedInt32 used = CheckedInt32(aWidth) * BytesPerPixel(aFormat); + if (!used.isValid() || used.value() < 0) { + return -1; + } + return aStride - used.value(); +} + bool PremultiplyData(const uint8_t* aSrc, int32_t aSrcStride, SurfaceFormat aSrcFormat, uint8_t* aDst, int32_t aDstStride, SurfaceFormat aDstFormat, const IntSize& aSize) { if (aSize.IsEmpty()) { return true; } IntSize size = CollapseSize(aSize, aSrcStride, aDstStride); // Find gap from end of row to the start of the next row. - int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width; - int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width; + int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride); + int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride); MOZ_ASSERT(srcGap >= 0 && dstGap >= 0); + if (srcGap < 0 || dstGap < 0) { + return false; + } #define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size) #ifdef USE_SSE2 switch (FORMAT_KEY(aSrcFormat, aDstFormat)) { PREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::B8G8R8A8) PREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::B8G8R8X8) PREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8A8) @@ -399,19 +413,22 @@ UnpremultiplyData(const uint8_t* aSrc, i uint8_t* aDst, int32_t aDstStride, SurfaceFormat aDstFormat, const IntSize& aSize) { if (aSize.IsEmpty()) { return true; } IntSize size = CollapseSize(aSize, aSrcStride, aDstStride); // Find gap from end of row to the start of the next row. - int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width; - int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width; + int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride); + int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride); MOZ_ASSERT(srcGap >= 0 && dstGap >= 0); + if (srcGap < 0 || dstGap < 0) { + return false; + } #define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size) #ifdef USE_SSE2 switch (FORMAT_KEY(aSrcFormat, aDstFormat)) { UNPREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::B8G8R8A8) UNPREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8A8) UNPREMULTIPLY_SSE2(SurfaceFormat::R8G8B8A8, SurfaceFormat::R8G8B8A8) @@ -697,19 +714,22 @@ SwizzleData(const uint8_t* aSrc, int32_t uint8_t* aDst, int32_t aDstStride, SurfaceFormat aDstFormat, const IntSize& aSize) { if (aSize.IsEmpty()) { return true; } IntSize size = CollapseSize(aSize, aSrcStride, aDstStride); // Find gap from end of row to the start of the next row. - int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width; - int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width; + int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride); + int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride); MOZ_ASSERT(srcGap >= 0 && dstGap >= 0); + if (srcGap < 0 || dstGap < 0) { + return false; + } #define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size) #ifdef USE_SSE2 switch (FORMAT_KEY(aSrcFormat, aDstFormat)) { SWIZZLE_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8A8) SWIZZLE_SSE2(SurfaceFormat::B8G8R8X8, SurfaceFormat::R8G8B8X8) SWIZZLE_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8X8)