[Samba] Re: smbclient and large file support
Adnan Olia
I made the changes with this patch on Samba 2.2.7a and then re-compiled it
with
./configure --with-libsmbclient
After the new samba install, I tried to "get" a file from Win2000 Advanced
Server to Mandrake Linux 8.2 server using SMBCLIENT and was successfully
able to transfer a 6.63GB file. Then I tried to "put" a file from Mandrake
Linux 8.2 server to the Win2000 Advanced Server and the file completed at
4GB rather than 6.63 GB. Is this normal?
Did I do something wrong or is there some other patch. Please advise
Thanks
Adnan Olia
"Herb Lewis" <he...@sgi.com> wrote in message
news:3E026261...@sgi.com...
> smbclient (and smbtar) in version 2.2.7a (and prior) has problems with
> large files (> 4GB). The following patch (against 2.2.7a) fixes all
> known problems with this. This code has been checked into the CVS tree
> in all branches as well.
>
> --
> ======================================================================
> Herb Lewis Silicon Graphics
> Networking Engineer 1600 Amphitheatre Pkwy MS-510
> Strategic Software Organization Mountain View, CA 94043-1351
> he...@sgi.com Tel: 650-933-2177
> http://www.sgi.com Fax: 650-932-2177
> PGP Key: 0x8408D65D
> ======================================================================
----------------------------------------------------------------------------
----
> --- samba-2.2.7a/source/client/client.c Wed Dec 4 09:16:34 2002
> +++ samba-2.2.7a-fixed/source/client/client.c Thu Dec 19 15:41:51 2002
> @@ -92,9 +92,9 @@
>
> /* timing globals */
> off_t get_total_size = 0;
> -int get_total_time_ms = 0;
> +unsigned int get_total_time_ms = 0;
> off_t put_total_size = 0;
> -int put_total_time_ms = 0;
> +unsigned int put_total_time_ms = 0;
>
> /* totals globals */
> static double dir_total;
> --- samba-2.2.7a/source/client/clitar.c Tue Apr 30 06:26:18 2002
> +++ samba-2.2.7a-fixed/source/client/clitar.c Thu Dec 19 15:50:20 2002
> @@ -45,10 +45,10 @@
>
> struct file_info_struct
> {
> - size_t size;
> + SMB_BIG_UINT size;
> uint16 mode;
> - int uid;
> - int gid;
> + uid_t uid;
> + gid_t gid;
> /* These times are normally kept in GMT */
> time_t mtime;
> time_t atime;
> @@ -125,11 +125,11 @@
> int blocksize=20;
> int tarhandle;
>
> -static void writetarheader(int f, char *aname, int size, time_t mtime,
> +static void writetarheader(int f, char *aname, SMB_BIG_UINT size, time_t
mtime,
> char *amode, unsigned char ftype);
> static void do_atar(char *rname,char *lname,file_info *finfo1);
> static void do_tar(file_info *finfo);
> -static void oct_it(long value, int ndgs, char *p);
> +static void oct_it(SMB_BIG_UINT value, int ndgs, char *p);
> static void fixtarname(char *tptr, char *fp, int l);
> static int dotarbuf(int f, char *b, int n);
> static void dozerobuf(int f, int n);
> @@ -168,7 +168,7 @@
>
/***************************************************************************
*
> Write a tar header to buffer
>
****************************************************************************
/
> -static void writetarheader(int f, char *aname, int size, time_t mtime,
> +static void writetarheader(int f, char *aname, SMB_BIG_UINT size, time_t
mtime,
> char *amode, unsigned char ftype)
> {
> union hblock hb;
> @@ -175,7 +175,7 @@
> int i, chk, l;
> char *jp;
>
> - DEBUG(5, ("WriteTarHdr, Type = %c, Size= %i, Name = %s\n", ftype, size,
aname));
> + DEBUG(5, ("WriteTarHdr, Type = %c, Size= %.0f, Name = %s\n", ftype,
(double)size, aname));
>
> memset(hb.dummy, 0, sizeof(hb.dummy));
>
> @@ -207,10 +207,10 @@
>
> hb.dbuf.name[NAMSIZ-1]='\0';
> safe_strcpy(hb.dbuf.mode, amode, strlen(amode));
> - oct_it(0L, 8, hb.dbuf.uid);
> - oct_it(0L, 8, hb.dbuf.gid);
> - oct_it((long) size, 13, hb.dbuf.size);
> - oct_it((long) mtime, 13, hb.dbuf.mtime);
> + oct_it((SMB_BIG_UINT)0, 8, hb.dbuf.uid);
> + oct_it((SMB_BIG_UINT)0, 8, hb.dbuf.gid);
> + oct_it((SMB_BIG_UINT) size, 13, hb.dbuf.size);
> + oct_it((SMB_BIG_UINT) mtime, 13, hb.dbuf.mtime);
> memcpy(hb.dbuf.chksum, " ", sizeof(hb.dbuf.chksum));
> memset(hb.dbuf.linkname, 0, NAMSIZ);
> hb.dbuf.linkflag=ftype;
> @@ -217,7 +217,7 @@
>
> for (chk=0, i=sizeof(hb.dummy), jp=hb.dummy; --i>=0;) chk+=(0xFF &
*jp++);
>
> - oct_it((long) chk, 8, hb.dbuf.chksum);
> + oct_it((SMB_BIG_UINT) chk, 8, hb.dbuf.chksum);
> hb.dbuf.chksum[6] = '\0';
>
> (void) dotarbuf(f, hb.dummy, sizeof(hb.dummy));
> @@ -450,7 +450,7 @@
>
/***************************************************************************
*
> Convert from decimal to octal string
>
****************************************************************************
/
> -static void oct_it (long value, int ndgs, char *p)
> +static void oct_it (SMB_BIG_UINT value, int ndgs, char *p)
> {
> /* Converts long to octal string, pads with leading zeros */
>
> @@ -621,7 +621,7 @@
> static void do_atar(char *rname,char *lname,file_info *finfo1)
> {
> int fnum;
> - uint32 nread=0;
> + SMB_BIG_UINT nread=0;
> char ftype;
> file_info2 finfo;
> BOOL close_done = False;
> @@ -643,6 +643,7 @@
> finfo.mtime = finfo1 -> mtime;
> finfo.atime = finfo1 -> atime;
> finfo.ctime = finfo1 -> ctime;
> + finfo.name = finfo1 -> name;
> }
> else {
> finfo.size = def_finfo.size;
> @@ -652,13 +653,14 @@
> finfo.mtime = def_finfo.mtime;
> finfo.atime = def_finfo.atime;
> finfo.ctime = def_finfo.ctime;
> + finfo.name = def_finfo.name;
> }
>
> if (dry_run)
> {
> - DEBUG(3,("skipping file %s of size %d bytes\n",
> + DEBUG(3,("skipping file %s of size %12.0f bytes\n",
> finfo.name,
> - (int)finfo.size));
> + (double)finfo.size));
> shallitime=0;
> ttarf+=finfo.size + TBLOCK - (finfo.size % TBLOCK);
> ntarf++;
> @@ -709,9 +711,9 @@
> }
> else
> {
> - DEBUG(3,("getting file %s of size %d bytes as a tar file %s",
> + DEBUG(3,("getting file %s of size %.0f bytes as a tar file %s",
> finfo.name,
> - (int)finfo.size,
> + (double)finfo.size,
> lname));
>
> /* write a tar header, don't bother with mode - just set to 100644
*/
> @@ -719,7 +721,7 @@
>
> while (nread < finfo.size && !close_done) {
>
> - DEBUG(3,("nread=%d\n",nread));
> + DEBUG(3,("nread=%.0f\n",(double)nread));
>
> datalen = cli_read(cli, fnum, data, nread, read_size);
>
> @@ -736,7 +738,7 @@
>
> if (nread > finfo.size) {
> datalen -= nread - finfo.size;
> - DEBUG(0,("File size change - truncating %s to %d bytes\n", finfo.name,
(int)finfo.size));
> + DEBUG(0,("File size change - truncating %s to %.0f bytes\n", finfo.name,
(double)finfo.size));
> }
>
> /* add received bits of file to buffer - dotarbuf will
> @@ -756,7 +758,7 @@
>
> /* pad tar file with zero's if we couldn't get entire file */
> if (nread < finfo.size) {
> - DEBUG(0, ("Didn't get entire file. size=%d, nread=%d\n",
(int)finfo.size, (int)nread));
> + DEBUG(0, ("Didn't get entire file. size=%.0f, nread=%d\n",
(double)finfo.size, (int)nread));
> if (padit(data, sizeof(data), finfo.size - nread))
> DEBUG(0,("Error writing tar file - %s\n", strerror(errno)));
> }
> @@ -789,8 +791,8 @@
>
> if (tar_noisy)
> {
> - DEBUG(0, ("%10d (%7.1f kb/s) %s\n",
> - (int)finfo.size, finfo.size / MAX(0.001, (1.024*this_time)),
> + DEBUG(0, ("%12.0f (%7.1f kb/s) %s\n",
> + (double)finfo.size, finfo.size / MAX(0.001, (1.024*this_time)),
> finfo.name));
> }
>
> @@ -1868,7 +1870,7 @@
> if (tar_type=='c' && (dry_run || strcmp(argv[Optind],
"/dev/null")==0))
> {
> if (!dry_run) {
> - DEBUG(0,("Output is /dev/null, assuming dry_run"));
> + DEBUG(0,("Output is /dev/null, assuming dry_run\n"));
> dry_run = True;
> }
> tarhandle=-1;
> --- samba-2.2.7a/source/lib/snprintf.c Wed Jun 5 12:31:29 2002
> +++ samba-2.2.7a-fixed/source/lib/snprintf.c Thu Dec 19 15:42:00 2002
> @@ -341,6 +341,7 @@
> fvalue = va_arg (args, LDOUBLE);
> else
> fvalue = va_arg (args, double);
> + fmtfp (buffer, &currlen, maxlen, fvalue, min, max, flags);
> break;
> case 'G':
> flags |= DP_F_UP;
> @@ -349,6 +350,7 @@
> fvalue = va_arg (args, LDOUBLE);
> else
> fvalue = va_arg (args, double);
> + fmtfp (buffer, &currlen, maxlen, fvalue, min, max, flags);
> break;
> case 'c':
> dopr_outch (buffer, &currlen, maxlen, va_arg (args, int));
> --- samba-2.2.7a/source/libsmb/clireadwrite.c Tue Apr 30 06:27:03 2002
> +++ samba-2.2.7a-fixed/source/libsmb/clireadwrite.c Thu Dec 19 15:48:47
2002
> @@ -30,10 +30,15 @@
> static BOOL cli_issue_read(struct cli_state *cli, int fnum, off_t offset,
> size_t size, int i)
> {
> + BOOL bigoffset = False;
> +
> memset(cli->outbuf,'\0',smb_size);
> memset(cli->inbuf,'\0',smb_size);
>
> - set_message(cli->outbuf,10,0,True);
> + if ((SMB_BIG_UINT)offset >> 32)
> + bigoffset = True;
> +
> + set_message(cli->outbuf,bigoffset ? 12 : 10,0,True);
>
> SCVAL(cli->outbuf,smb_com,SMBreadX);
> SSVAL(cli->outbuf,smb_tid,cli->cnum);
> @@ -45,6 +50,9 @@
> SSVAL(cli->outbuf,smb_vwv5,size);
> SSVAL(cli->outbuf,smb_vwv6,size);
> SSVAL(cli->outbuf,smb_mid,cli->mid + i);
> +
> + if (bigoffset)
> + SIVAL(cli->outbuf,smb_vwv10,(offset>>32) & 0xffffffff);
>
> return cli_send_smb(cli);
> }
>
--
To unsubscribe from this list go to the following URL and read the
instructions: http://lists.samba.org/mailman/listinfo/samba
Richard Sharpe
> Hi Herb,
>
> I made the changes with this patch on Samba 2.2.7a and then re-compiled it
> with
> ./configure --with-libsmbclient
>
> After the new samba install, I tried to "get" a file from Win2000 Advanced
> Server to Mandrake Linux 8.2 server using SMBCLIENT and was successfully
> able to transfer a 6.63GB file. Then I tried to "put" a file from Mandrake
> Linux 8.2 server to the Win2000 Advanced Server and the file completed at
> 4GB rather than 6.63 GB. Is this normal?
>
> Did I do something wrong or is there some other patch. Please advise
I wonder at the ability of cli_write to handle large offsets.
Regards
-----
Richard Sharpe, rsharpe[at]ns.aus.com, rsharpe[at]samba.org,
sharpe[at]ethereal.com, http://www.richardsharpe.com
Richard Sharpe
> Hi Herb,
>
> I made the changes with this patch on Samba 2.2.7a and then re-compiled it
> with
> ./configure --with-libsmbclient
>
> After the new samba install, I tried to "get" a file from Win2000 Advanced
> Server to Mandrake Linux 8.2 server using SMBCLIENT and was successfully
> able to transfer a 6.63GB file. Then I tried to "put" a file from Mandrake
> Linux 8.2 server to the Win2000 Advanced Server and the file completed at
> 4GB rather than 6.63 GB. Is this normal?
>
> Did I do something wrong or is there some other patch. Please advise
OK, I have confirmed that this occurs with smbclient talking to Samba
3.0alpha.
Now testing a get as well.
Richard Sharpe
> Hi Herb,
>
> I made the changes with this patch on Samba 2.2.7a and then re-compiled it
> with
> ./configure --with-libsmbclient
>
> After the new samba install, I tried to "get" a file from Win2000 Advanced
> Server to Mandrake Linux 8.2 server using SMBCLIENT and was successfully
> able to transfer a 6.63GB file. Then I tried to "put" a file from Mandrake
> Linux 8.2 server to the Win2000 Advanced Server and the file completed at
> 4GB rather than 6.63 GB. Is this normal?
OK, I have finally found the problem :-)
cli_issue_write, in libsmb/clireadwrite.c has a bug.
It does not properly code the WriteX SMB. If the offset is larger than
32-bit, it should code the WordCount as 14 and should include the upper 32
bits of the offset later in the parameters portion of the SMB.
Since this is not done, all offsets are 32-bits and anything beyond 4GB is
written to the segment just before or beyond 4GB :-)
The fix will take a small while to code.
Richard Sharpe
> Hi Herb,
>
> I made the changes with this patch on Samba 2.2.7a and then re-compiled it
> with
> ./configure --with-libsmbclient
>
> After the new samba install, I tried to "get" a file from Win2000 Advanced
> Server to Mandrake Linux 8.2 server using SMBCLIENT and was successfully
> able to transfer a 6.63GB file. Then I tried to "put" a file from Mandrake
> Linux 8.2 server to the Win2000 Advanced Server and the file completed at
> 4GB rather than 6.63 GB. Is this normal?
The following might be a fix to your problem :-)
diff -u -r1.2.4.9 clireadwrite.c
--- libsmb/clireadwrite.c 19 Dec 2002 16:12:41 -0000 1.2.4.9
+++ libsmb/clireadwrite.c 30 Dec 2002 04:04:37 -0000
@@ -231,6 +231,7 @@
size_t size, int i)
{
char *p;
+ BOOL bigoffset = False;
if (size > cli->bufsize) {
cli->outbuf = realloc(cli->outbuf, size + 1024);
@@ -243,7 +244,10 @@
memset(cli->outbuf,'\0',smb_size);
memset(cli->inbuf,'\0',smb_size);
- if (size > 0xFFFF)
+ if ((SMB_BIG_UINT)offset >> 32)
+ bigoffset = True;
+
+ if (bigoffset)
set_message(cli->outbuf,14,0,True);
else
set_message(cli->outbuf,12,0,True);
@@ -256,14 +260,20 @@
SSVAL(cli->outbuf,smb_vwv2,fnum);
SIVAL(cli->outbuf,smb_vwv3,offset);
- SIVAL(cli->outbuf,smb_vwv5,(mode & 0x0008) ? 0xFFFFFFFF : 0);
+ SIVAL(cli->outbuf,smb_vwv5,0);
SSVAL(cli->outbuf,smb_vwv7,mode);
+ /*
+ * THe following is still wrong ...
+ */
SSVAL(cli->outbuf,smb_vwv8,(mode & 0x0008) ? size : 0);
SSVAL(cli->outbuf,smb_vwv9,((size>>16)&1));
SSVAL(cli->outbuf,smb_vwv10,size);
SSVAL(cli->outbuf,smb_vwv11,
smb_buf(cli->outbuf) - smb_base(cli->outbuf));
+
+ if (bigoffset)
+ SIVAL(cli->outbuf,smb_vwv12,(offset>>32) & 0xffffffff);
p = smb_base(cli->outbuf) + SVAL(cli->outbuf,smb_vwv11);
memcpy(p, buf, size);