GetGlyphOutlineW returned dataSize is not based on directly on gmBlackBoxY

[GoogleCodeExporter]

Getting assert:
  FOG_ASSERT(dataSize == bitmapd->stride * bitmapd->height);

As bitmap not allocated to correct size based on dataSize returned
from GetGlyphOutlineW.

Code fix is below.

GlyphData* WinFontFace::renderGlyph(HDC hdc, uint32_t uc)
{
  // renderBegin() must be called before
  FOG_ASSERT(hdc);

  GlyphData* glyphd = NULL;
  ImageData* bitmapd = NULL;

  GLYPHMETRICS gm;
  ZeroMemory(&gm, sizeof(gm));

  MAT2 mat2;

  if (transform.isIdentity())
  {
    mat2 = mat2identity;
  }
  else
  {
    mat2.eM11 = FloatToFIXED(transform._00);
    mat2.eM12 = FloatToFIXED(transform._01);
    mat2.eM21 = FloatToFIXED(transform._10);
    mat2.eM22 = FloatToFIXED(transform._11);
  }

  uint32_t dataSize = GetGlyphOutlineW(hdc, uc, GGO_GRAY8_BITMAP, &gm, 0, NULL, &mat2);

  // GetGlyphOutline() fails when being called for GGO_GRAY8_BITMAP and white space.
  if (dataSize == GDI_ERROR)
  {
    dataSize = GetGlyphOutlineW(hdc, uc, GGO_METRICS, &gm, 0, NULL, &mat2);
    if (dataSize == GDI_ERROR) return NULL;
    dataSize = 0;
  }

  glyphd = fog_new GlyphData();
  if (glyphd == NULL) return NULL;

  // Whitespace?
  if (dataSize == 0) gm.gmBlackBoxX = gm.gmBlackBoxY = 0;

  glyphd->offset.set(gm.gmptGlyphOrigin.x, (int)metrics.getAscent() - gm.gmptGlyphOrigin.y);
  glyphd->beginWidth = 0;
  glyphd->endWidth = 0;

  glyphd->advance = gm.gmCellIncX;
  // glyphd->advanceY = -gm.gmCellIncY;

  // Whitespace? We are done
  if (dataSize == 0) return glyphd;

  // dataSize returned by GetGlyphOutlineW is not always based on gm.bmBlackBoxY
  // so wee need to get the correct height using the stride and the dataSize returned by GetGlyphOutlineW
  int stride = (gm.gmBlackBoxX + 3) & ~3 ;
  int bmheight = dataSize / stride ;

  // Alloc image for glyph
  if (glyphd->bitmap.create(gm.gmBlackBoxX, bmheight, IMAGE_FORMAT_A8) != ERR_OK)
  {
    fog_delete(glyphd);
    return NULL;
  }
  bitmapd = glyphd->bitmap._d;

  // Fog library should align scanlines to 32 bits like Windows does.
  FOG_ASSERT((bitmapd->stride & 0x3) == 0);
  // This should be also equal.
  FOG_ASSERT(dataSize == bitmapd->stride * bitmapd->height);

  dataSize = GetGlyphOutlineW(hdc, uc, GGO_GRAY8_BITMAP, &gm, dataSize, bitmapd->data, &mat2);
  // If previous call to GetGlyphOutlineW was ok, this should be also ok, but nobody knows.
  if (dataSize == GDI_ERROR)
  {
    fog_delete(glyphd);
    return NULL;
  }

  // Fog is using 256 level of antialiasing so extend the glyph provided by
  // Windows (that uses only 64 levels).
  uint32_t x, y;
  for (y = 0; y != gm.gmBlackBoxY; y++)
  {
    uint8_t* p = bitmapd->first + y * bitmapd->stride;
    for (x = 0; x < gm.gmBlackBoxX; x++)
    {
      uint8_t p0 = p[0];
      *p++ = (p0 > 63) ? (0xFF) : (p0 << 2) | (p0 & 0x03);
    }
  }

  return glyphd;
}

Original issue reported on code.google.com by marietta...@peernet.com on 1 Nov 2010 at 12:56

[GoogleCodeExporter]

Thanks! Your fix will be merged with trunk

Original comment by kobalicek.petr on 13 Nov 2010 at 2:52

• Changed state: Accepted

[GoogleCodeExporter]

I fixed this in my working copy, will be commited soon, leaving 'Accepted'.

Original comment by kobalicek.petr on 24 Nov 2010 at 10:43