>Number:         6163
>Category:       pkg
>Synopsis:       VNC package has memory leak in Xvnc
>Confidential:   no
>Severity:       serious
>Priority:       medium
>Responsible:    gnats-admin (GNATS administrator)
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Tue Sep 15 13:35:01 1998
>Last-Modified:
>Originator:     J Rizzo
>Organization:
Jeff Rizzo                                         http://boogers.sf.ca.us/~riz
>Release:        NetBSD-current packages as of 14September1998
>Environment:
	
System: NetBSD boogers.sf.ca.us 1.3 NetBSD 1.3 (BOOGERS) #0: Sat Feb 21 14:32:20 PST 1998 riz@boogers.sf.ca.us:/work/tmp/netbsd-1.3/src/sys/arch/sparc/compile/BOOGERS sparc


>Description:
	There is a memory leak in the rfbserver code for the VNC package.
	The author has provided a patch, below.

>How-To-Repeat:
	Run Xvnc for a long time.
>Fix:
	install the following as /usr/pkgsrc/net/vnc/patches/patch-ad

---
*** Xvnc/programs/Xserver/hw/vnc/rfbserver.c.orig	Mon Sep 14 20:18:30 1998
--- Xvnc/programs/Xserver/hw/vnc/rfbserver.c	Mon Sep 14 20:22:52 1998
***************
*** 816,825 ****
      ublen = sz_rfbFramebufferUpdateMsg;
  
      if (REGION_NOTEMPTY(pScreen,&updateCopyRegion)) {
! 	if (!rfbSendCopyRegion(cl,&updateCopyRegion,dx,dy))
  	    return FALSE;
      }
  
      for (i = 0; i < REGION_NUM_RECTS(&updateRegion); i++) {
  	int x = REGION_RECTS(&updateRegion)[i].x1;
  	int y = REGION_RECTS(&updateRegion)[i].y1;
--- 816,830 ----
      ublen = sz_rfbFramebufferUpdateMsg;
  
      if (REGION_NOTEMPTY(pScreen,&updateCopyRegion)) {
! 	if (!rfbSendCopyRegion(cl,&updateCopyRegion,dx,dy)) {
! 	    REGION_UNINIT(pScreen,&updateRegion);
! 	    REGION_UNINIT(pScreen,&updateCopyRegion);
  	    return FALSE;
+ 	}
      }
  
+     REGION_UNINIT(pScreen,&updateCopyRegion);
+ 
      for (i = 0; i < REGION_NUM_RECTS(&updateRegion); i++) {
  	int x = REGION_RECTS(&updateRegion)[i].x1;
  	int y = REGION_RECTS(&updateRegion)[i].y1;
***************
*** 831,854 ****
  
  	switch (cl->preferredEncoding) {
  	case rfbEncodingRaw:
! 	    if (!rfbSendRectEncodingRaw(cl, x, y, w, h))
  		return FALSE;
  	    break;
  	case rfbEncodingRRE:
! 	    if (!rfbSendRectEncodingRRE(cl, x, y, w, h))
  		return FALSE;
  	    break;
  	case rfbEncodingCoRRE:
! 	    if (!rfbSendRectEncodingCoRRE(cl, x, y, w, h))
  		return FALSE;
  	    break;
  	case rfbEncodingHextile:
! 	    if (!rfbSendRectEncodingHextile(cl, x, y, w, h))
  		return FALSE;
  	    break;
  	}
      }
  
      if (!rfbSendUpdateBuf(cl))
  	return FALSE;
  
--- 836,869 ----
  
  	switch (cl->preferredEncoding) {
  	case rfbEncodingRaw:
! 	    if (!rfbSendRectEncodingRaw(cl, x, y, w, h)) {
! 		REGION_UNINIT(pScreen,&updateRegion);
  		return FALSE;
+ 	    }
  	    break;
  	case rfbEncodingRRE:
! 	    if (!rfbSendRectEncodingRRE(cl, x, y, w, h)) {
! 		REGION_UNINIT(pScreen,&updateRegion);
  		return FALSE;
+ 	    }
  	    break;
  	case rfbEncodingCoRRE:
! 	    if (!rfbSendRectEncodingCoRRE(cl, x, y, w, h)) {
! 		REGION_UNINIT(pScreen,&updateRegion);
  		return FALSE;
+ 	    }
  	    break;
  	case rfbEncodingHextile:
! 	    if (!rfbSendRectEncodingHextile(cl, x, y, w, h)) {
! 		REGION_UNINIT(pScreen,&updateRegion);
  		return FALSE;
+ 	    }
  	    break;
  	}
      }
  
+     REGION_UNINIT(pScreen,&updateRegion);
+ 
      if (!rfbSendUpdateBuf(cl))
  	return FALSE;
  
***************
*** 896,904 ****
  	firstInNextBand = thisRect;
  	nrectsInBand = 0;
  
! 	while ((REGION_RECTS(reg)[firstInNextBand].y1
! 		== REGION_RECTS(reg)[thisRect].y1) &&
! 	       (nrects > 0))
  	{
  	    firstInNextBand += y_inc;
  	    nrects--;
--- 911,919 ----
  	firstInNextBand = thisRect;
  	nrectsInBand = 0;
  
! 	while ((nrects > 0) &&
! 	       (REGION_RECTS(reg)[firstInNextBand].y1
! 		== REGION_RECTS(reg)[thisRect].y1))
  	{
  	    firstInNextBand += y_inc;
  	    nrects--;
>Audit-Trail:
>Unformatted: