Subject: kern/30823: Panic reading files larger than 4GB on NTFS
To: None <kern-bug-people@netbsd.org, gnats-admin@netbsd.org,>
From: Dave Huang <khym@azeotrope.org>
List: netbsd-bugs
Date: 07/24/2005 22:21:00
>Number: 30823
>Category: kern
>Synopsis: Panic reading files larger than 4GB on NTFS
>Confidential: no
>Severity: serious
>Priority: medium
>Responsible: kern-bug-people
>State: open
>Class: sw-bug
>Submitter-Id: net
>Arrival-Date: Sun Jul 24 22:21:00 +0000 2005
>Originator: Dave Huang
>Release: NetBSD 3.99.7
>Organization:
>Environment:
System: NetBSD cheetah.azeotrope.org 3.99.7 NetBSD 3.99.7 (CHEETAH) #10: Sun Jul 24 16:42:20 CDT 2005 khym@cheetah.azeotrope.org:/usr/obj.amd64/sys/arch/amd64/compile/CHEETAH amd64
Architecture: x86_64
Machine: amd64
>Description:
The NTFS implementation has trouble with files larger than 4GB
(maybe even with files larger than 2GB?). Reading past the 4GB point
causes a panic:
panic: buf mem pool index 54
Stopped in pid 26.1 (lav2yuv) at netbsd:cpu_Debugger+0x5: leave
db{0}> t
cpu_Debugger() at netbsd:cpu_Debugger+0x5
panic() at netbsd:panic+0x1c8
allocbuf() at netbsd:allocbuf+0x296
getblk() at netbsd:getblk+0x201
bio_doread() at netbsd:bio_doread+0x49
bread() at netbsd:bread+0x17
ntfs_readntvattr_plain() at netbsd:ntfs_readntvattr_plain+0x214
ntfs_readattr_plain() at netbsd:ntfs_readattr_plain+0xed
ntfs_readattr() at netbsd:ntfs_readattr+0x8bf
ntfs_read() at netbsd:ntfs_read+0x70
VOP_READ() at netbsd:VOP_READ+0x31
vn_read() at netbsd:vn_read+0x9a
dofileread() at netbsd:dofileread+0x92
sys_read() at netbsd:sys_read+0x75
syscall_fancy() at netbsd:syscall_fancy+0x125
uvm_fault(0xffff80000e188420, 0x0, 0, 1) -> e
kernel: page fault trap, code=0
Faulted in DDB; continuing...
db{0}>
>How-To-Repeat:
Create a file larger than 4GB on an NTFS partition with
Windows (I used a 10GB file, but I think anything over 4GB would show
the problem), then run:
% dd if=largefile of=/dev/null bs=1m count=1 skip=5000
>Fix:
There are a few places where a 64-bit quantity is being stored in a
32-bit variable. No idea why gcc doesn't warn about it:
Index: ntfs_subr.c
===================================================================
RCS file: /cvsroot/src/sys/fs/ntfs/ntfs_subr.c,v
retrieving revision 1.16
diff -u -r1.16 ntfs_subr.c
--- ntfs_subr.c 29 May 2005 21:00:29 -0000 1.16
+++ ntfs_subr.c 24 Jul 2005 21:49:15 -0000
@@ -1479,7 +1479,7 @@
struct uio *uio)
{
int error = 0;
- int off;
+ off_t off;
int cnt;
cn_t ccn, ccl, cn, left, cl;
caddr_t data = rdata;
@@ -1589,7 +1589,7 @@
struct uio *uio)
{
int error = 0;
- int off;
+ off_t off;
*initp = 0;
if (vap->va_flag & NTFS_AF_INRUN) {
Index: ntfs_subr.h
===================================================================
RCS file: /cvsroot/src/sys/fs/ntfs/ntfs_subr.h,v
retrieving revision 1.2
diff -u -r1.2 ntfs_subr.h
--- ntfs_subr.h 29 May 2005 21:00:29 -0000 1.2
+++ ntfs_subr.h 24 Jul 2005 21:49:16 -0000
@@ -45,8 +45,8 @@
u_int32_t va_compression;
u_int32_t va_compressalg;
- u_int32_t va_datalen;
- u_int32_t va_allocated;
+ u_int64_t va_datalen;
+ u_int64_t va_allocated;
cn_t va_vcnstart;
cn_t va_vcnend;
u_int16_t va_index;
Also, in many printf()s (mainly NTFS_DEBUG ones), 64-bit quantities
are cast to 32-bit, which makes debugging difficult, since the debug
printfs are showing bogus values. As recommended by the style guide,
here's a patch to cast 64-bit quantities to long long for printing.
Index: ntfs_subr.c
===================================================================
RCS file: /cvsroot/src/sys/fs/ntfs/ntfs_subr.c,v
retrieving revision 1.16
diff -u -r1.16 ntfs_subr.c
--- ntfs_subr.c 29 May 2005 21:00:29 -0000 1.16
+++ ntfs_subr.c 24 Jul 2005 21:48:33 -0000
@@ -137,9 +137,9 @@
*lvapp = NULL;
*vapp = NULL;
for (vap = ip->i_valist.lh_first; vap; vap = vap->va_list.le_next) {
- ddprintf(("ntfs_findvattr: type: 0x%x, vcn: %d - %d\n", \
- vap->va_type, (u_int32_t) vap->va_vcnstart, \
- (u_int32_t) vap->va_vcnend));
+ ddprintf(("ntfs_findvattr: type: 0x%x, vcn: %qu - %qu\n", \
+ vap->va_type, (long long) vap->va_vcnstart, \
+ (long long) vap->va_vcnend));
if ((vap->va_type == type) &&
(vap->va_vcnstart <= vcn) && (vap->va_vcnend >= vcn) &&
(vap->va_namelen == namelen) &&
@@ -184,13 +184,13 @@
if (name) {
dprintf(("ntfs_ntvattrget: " \
- "ino: %d, type: 0x%x, name: %s, vcn: %d\n", \
- ip->i_number, type, name, (u_int32_t) vcn));
+ "ino: %d, type: 0x%x, name: %s, vcn: %qu\n", \
+ ip->i_number, type, name, (long long) vcn));
namelen = strlen(name);
} else {
dprintf(("ntfs_ntvattrget: " \
- "ino: %d, type: 0x%x, vcn: %d\n", \
- ip->i_number, type, (u_int32_t) vcn));
+ "ino: %d, type: 0x%x, vcn: %qu\n", \
+ ip->i_number, type, (long long) vcn));
name = "";
namelen = 0;
}
@@ -201,8 +201,8 @@
if (!lvap) {
dprintf(("ntfs_ntvattrget: UNEXISTED ATTRIBUTE: " \
- "ino: %d, type: 0x%x, name: %s, vcn: %d\n", \
- ip->i_number, type, name, (u_int32_t) vcn));
+ "ino: %d, type: 0x%x, name: %s, vcn: %qu\n", \
+ ip->i_number, type, name, (long long) vcn));
return (ENOENT);
}
/* Scan $ATTRIBUTE_LIST for requested attribute */
@@ -218,9 +218,9 @@
for(; len > 0; aalp = nextaalp) {
dprintf(("ntfs_ntvattrget: " \
- "attrlist: ino: %d, attr: 0x%x, vcn: %d\n", \
+ "attrlist: ino: %d, attr: 0x%x, vcn: %qu\n", \
aalp->al_inumber, aalp->al_type, \
- (u_int32_t) aalp->al_vcnstart));
+ (long long) aalp->al_vcnstart));
if (len > aalp->reclen) {
nextaalp = NTFS_NEXTREC(aalp, struct attr_attrlist *);
@@ -260,8 +260,8 @@
error = ENOENT;
dprintf(("ntfs_ntvattrget: UNEXISTED ATTRIBUTE: " \
- "ino: %d, type: 0x%x, name: %.*s, vcn: %d\n", \
- ip->i_number, type, (int) namelen, name, (u_int32_t) vcn));
+ "ino: %d, type: 0x%x, name: %.*s, vcn: %qu\n", \
+ ip->i_number, type, (int) namelen, name, (long long) vcn));
out:
free(alpool, M_TEMP);
return (error);
@@ -595,7 +595,7 @@
memcpy(vap->va_datap, (caddr_t) rap + rap->a_r.a_dataoff,
rap->a_r.a_datalen);
}
- ddprintf((", len: %d", vap->va_datalen));
+ ddprintf((", len: %qu", (long long)vap->va_datalen));
if (error)
FREE(vap, M_NTFSNTVATTR);
@@ -1221,8 +1221,8 @@
goto fail;
}
cpbl = ntfs_btocn(blsize + ntfs_cntob(1) - 1);
- dprintf(("ntfs_ntreaddir: indexalloc: %d, cpbl: %d\n",
- iavap->va_datalen, cpbl));
+ dprintf(("ntfs_ntreaddir: indexalloc: %qu, cpbl: %d\n",
+ (long long)iavap->va_datalen, cpbl));
} else {
dprintf(("ntfs_ntreadidir: w/o BitMap and IndexAllocation\n"));
iavap = bmvap = NULL;
@@ -1435,20 +1435,20 @@
if (error)
return (error);
towrite = MIN(left, ntfs_cntob(vap->va_vcnend + 1) - off);
- ddprintf(("ntfs_writeattr_plain: o: %d, s: %d (%d - %d)\n",
- (u_int32_t) off, (u_int32_t) towrite,
- (u_int32_t) vap->va_vcnstart,
- (u_int32_t) vap->va_vcnend));
+ ddprintf(("ntfs_writeattr_plain: o: %qd, s: %qd (%qu - %qu)\n",
+ (long long) off, (long long) towrite,
+ (long long) vap->va_vcnstart,
+ (long long) vap->va_vcnend));
error = ntfs_writentvattr_plain(ntmp, ip, vap,
off - ntfs_cntob(vap->va_vcnstart),
towrite, data, &init, uio);
if (error) {
dprintf(("ntfs_writeattr_plain: " \
- "ntfs_writentvattr_plain failed: o: %d, s: %d\n",
- (u_int32_t) off, (u_int32_t) towrite));
- dprintf(("ntfs_writeattr_plain: attrib: %d - %d\n",
- (u_int32_t) vap->va_vcnstart,
- (u_int32_t) vap->va_vcnend));
+ "ntfs_writentvattr_plain failed: o: %qd, s: %qd\n",
+ (long long) off, (long long) towrite));
+ dprintf(("ntfs_writeattr_plain: attrib: %qu - %qu\n",
+ (long long) vap->va_vcnstart,
+ (long long) vap->va_vcnend));
ntfs_ntvattrrele(vap);
break;
}
@@ -1506,9 +1506,9 @@
ccl = vap->va_vruncl[cnt];
ddprintf(("ntfs_writentvattr_plain: " \
- "left %d, cn: 0x%x, cl: %d, off: %d\n", \
- (u_int32_t) left, (u_int32_t) ccn, \
- (u_int32_t) ccl, (u_int32_t) off));
+ "left %qu, cn: 0x%qx, cl: %qu, off: %qd\n", \
+ (long long) left, (long long) ccn, \
+ (long long) ccl, (long long) off));
if (ntfs_cntob(ccl) < off) {
off -= ntfs_cntob(ccl);
@@ -1533,10 +1533,10 @@
cl = ntfs_btocl(tocopy + off);
KASSERT(cl == 1 && tocopy <= ntfs_cntob(1));
ddprintf(("ntfs_writentvattr_plain: write: " \
- "cn: 0x%x cl: %d, off: %d len: %d, left: %d\n",
- (u_int32_t) cn, (u_int32_t) cl,
- (u_int32_t) off, (u_int32_t) tocopy,
- (u_int32_t) left));
+ "cn: 0x%qx cl: %qu, off: %qd len: %qu, left: %qu\n",
+ (long long) cn, (long long) cl,
+ (long long) off, (long long) tocopy,
+ (long long) left));
if ((off == 0) && (tocopy == ntfs_cntob(cl)))
{
bp = getblk(ntmp->ntm_devvp, ntfs_cntobn(cn),
@@ -1612,9 +1612,9 @@
ccl = vap->va_vruncl[cnt];
ddprintf(("ntfs_readntvattr_plain: " \
- "left %d, cn: 0x%x, cl: %d, off: %d\n", \
- (u_int32_t) left, (u_int32_t) ccn, \
- (u_int32_t) ccl, (u_int32_t) off));
+ "left %qu, cn: 0x%qx, cl: %qu, off: %qd\n", \
+ (long long) left, (long long) ccn,
+ (long long) ccl, (long long) off));
if (ntfs_cntob(ccl) < off) {
off -= ntfs_cntob(ccl);
@@ -1641,13 +1641,13 @@
tocopy <= ntfs_cntob(1));
ddprintf(("ntfs_readntvattr_plain: " \
- "read: cn: 0x%x cl: %d, " \
- "off: %d len: %d, left: %d\n",
- (u_int32_t) cn,
- (u_int32_t) cl,
- (u_int32_t) off,
- (u_int32_t) tocopy,
- (u_int32_t) left));
+ "read: cn: 0x%qx cl: %qu, " \
+ "off: %qd len: %qu, left: %qu\n",
+ (long long) cn,
+ (long long) cl,
+ (long long) off,
+ (long long) tocopy,
+ (long long) left));
error = bread(ntmp->ntm_devvp,
ntfs_cntobn(cn),
ntfs_cntob(cl),
@@ -1674,11 +1674,11 @@
} else {
tocopy = MIN(left, ntfs_cntob(ccl) - off);
ddprintf(("ntfs_readntvattr_plain: "
- "hole: ccn: 0x%x ccl: %d, off: %d, " \
- " len: %d, left: %d\n",
- (u_int32_t) ccn, (u_int32_t) ccl,
- (u_int32_t) off, (u_int32_t) tocopy,
- (u_int32_t) left));
+ "hole: ccn: 0x%qx ccl: %qu, off: %qd, " \
+ " len: %qu, left: %qu\n",
+ (long long) ccn, (long long) ccl,
+ (long long) off, (long long) tocopy,
+ (long long) left));
left -= tocopy;
off = 0;
if (uio) {
@@ -1736,20 +1736,20 @@
if (error)
return (error);
toread = MIN(left, ntfs_cntob(vap->va_vcnend + 1) - off);
- ddprintf(("ntfs_readattr_plain: o: %d, s: %d (%d - %d)\n",
- (u_int32_t) off, (u_int32_t) toread,
- (u_int32_t) vap->va_vcnstart,
- (u_int32_t) vap->va_vcnend));
+ ddprintf(("ntfs_readattr_plain: o: %qd, s: %qd (%qu - %qu)\n",
+ (long long) off, (long long) toread,
+ (long long) vap->va_vcnstart,
+ (long long) vap->va_vcnend));
error = ntfs_readntvattr_plain(ntmp, ip, vap,
off - ntfs_cntob(vap->va_vcnstart),
toread, data, &init, uio);
if (error) {
printf("ntfs_readattr_plain: " \
- "ntfs_readntvattr_plain failed: o: %d, s: %d\n",
- (u_int32_t) off, (u_int32_t) toread);
- printf("ntfs_readattr_plain: attrib: %d - %d\n",
- (u_int32_t) vap->va_vcnstart,
- (u_int32_t) vap->va_vcnend);
+ "ntfs_readntvattr_plain failed: o: %qd, s: %qd\n",
+ (long long) off, (long long) toread);
+ printf("ntfs_readattr_plain: attrib: %qu - %qu\n",
+ (long long) vap->va_vcnstart,
+ (long long) vap->va_vcnend);
ntfs_ntvattrrele(vap);
break;
}
@@ -1781,8 +1781,8 @@
struct ntvattr *vap;
size_t init;
- ddprintf(("ntfs_readattr: reading %d: 0x%x, from %d size %d bytes\n",
- ip->i_number, attrnum, (u_int32_t) roff, (u_int32_t) rsize));
+ ddprintf(("ntfs_readattr: reading %d: 0x%x, from %qd size %qu bytes\n",
+ ip->i_number, attrnum, (long long) roff, (long long) rsize));
error = ntfs_ntvattrget(ntmp, ip, attrnum, attrname, 0, &vap);
if (error)
@@ -1790,9 +1790,9 @@
if ((roff > vap->va_datalen) ||
(roff + rsize > vap->va_datalen)) {
- printf("ntfs_readattr: offset too big: %ld (%ld) > %ld\n",
- (long int) roff, (long int) roff + rsize,
- (long int) vap->va_datalen);
+ printf("ntfs_readattr: offset too big: %qd (%qd) > %qu\n",
+ (long long) roff, (long long) (roff + rsize),
+ (long long) vap->va_datalen);
ntfs_ntvattrrele(vap);
return (E2BIG);
}
Index: ntfs_vnops.c
===================================================================
RCS file: /cvsroot/src/sys/fs/ntfs/ntfs_vnops.c,v
retrieving revision 1.21
diff -u -r1.21 ntfs_vnops.c
--- ntfs_vnops.c 26 Feb 2005 22:58:55 -0000 1.21
+++ ntfs_vnops.c 24 Jul 2005 21:48:34 -0000
@@ -156,9 +156,9 @@
u_int64_t toread;
int error;
- dprintf(("ntfs_read: ino: %d, off: %d resid: %d, segflg: %d\n",ip->i_number,(u_int32_t)uio->uio_offset,uio->uio_resid,uio->uio_segflg));
+ dprintf(("ntfs_read: ino: %d, off: %qd resid: %qd, segflg: %d\n",ip->i_number,(long long)uio->uio_offset,(long long)uio->uio_resid,uio->uio_segflg));
- dprintf(("ntfs_read: filesize: %d",(u_int32_t)fp->f_size));
+ dprintf(("ntfs_read: filesize: %qu",(long long)fp->f_size));
/* don't allow reading after end of file */
if (uio->uio_offset > fp->f_size)
@@ -166,7 +166,7 @@
else
toread = MIN(uio->uio_resid, fp->f_size - uio->uio_offset );
- dprintf((", toread: %d\n",(u_int32_t)toread));
+ dprintf((", toread: %qu\n",(long long)toread));
if (toread == 0)
return (0);
@@ -416,8 +416,8 @@
size_t written;
int error;
- dprintf(("ntfs_write: ino: %d, off: %d resid: %d, segflg: %d\n",ip->i_number,(u_int32_t)uio->uio_offset,uio->uio_resid,uio->uio_segflg));
- dprintf(("ntfs_write: filesize: %d",(u_int32_t)fp->f_size));
+ dprintf(("ntfs_write: ino: %d, off: %qd resid: %qd, segflg: %d\n",ip->i_number,(long long)uio->uio_offset,(long long)uio->uio_resid,uio->uio_segflg));
+ dprintf(("ntfs_write: filesize: %qu",(long long)fp->f_size));
if (uio->uio_resid + uio->uio_offset > fp->f_size) {
printf("ntfs_write: CAN'T WRITE BEYOND END OF FILE\n");
@@ -426,7 +426,7 @@
towrite = MIN(uio->uio_resid, fp->f_size - uio->uio_offset);
- dprintf((", towrite: %d\n",(u_int32_t)towrite));
+ dprintf((", towrite: %qu\n",(long long)towrite));
error = ntfs_writeattr_plain(ntmp, ip, fp->f_attrtype,
fp->f_attrname, uio->uio_offset, towrite, NULL, &written, uio);
@@ -586,7 +586,7 @@
struct dirent *cde;
off_t off;
- dprintf(("ntfs_readdir %d off: %d resid: %d\n",ip->i_number,(u_int32_t)uio->uio_offset,uio->uio_resid));
+ dprintf(("ntfs_readdir %d off: %qd resid: %qd\n",ip->i_number,(long long)uio->uio_offset,(long long)uio->uio_resid));
off = uio->uio_offset;
@@ -673,8 +673,8 @@
dprintf(("ntfs_readdir: %d entries (%d bytes) read\n",
ncookies,(u_int)(uio->uio_offset - off)));
- dprintf(("ntfs_readdir: off: %d resid: %d\n",
- (u_int32_t)uio->uio_offset,uio->uio_resid));
+ dprintf(("ntfs_readdir: off: %qd resid: %qu\n",
+ (long long)uio->uio_offset,(long long)uio->uio_resid));
if (!error && ap->a_ncookies != NULL) {
struct dirent* dpStart;
>Unformatted: