tool/-T: perform non-chunked transfer for stdin if it is a regular file

curl will now also compute the content-length of the transfer if stdin
is the file to upload and stdin is a regular file, using its file size.

Since, while being a regular file, stdin could not have its offset at
the start of the file, curl will now also get the current offset into
the upload file's file descriptor and use (filesize - offset) as
content-length for transfer instead of just using the full filesize.
This also fixes a bug on BSDs where open("/dev/fd/N") behaves like
dup(N), so, if N is a file descriptor to a regular file, the file offset
of the file descriptor returned by open() may not have been at the start
of the file despite curl's previous assumption.

Since I don't know anything about VMS systems, I left the behaviour for
VMS unchanged; on VMS, curl will still perform a chunked transfer if the
upload file is stdin.

I updated tests that were testing chunked transfers for stdin to use
<stdin pipe="yes"> since now curl only performs a chunked transfer by
default if the upload file (stdin or not) is not a regular file.

Fixes curl#12171
Fixes curl#12177

src/tool_operate.c

@@ -267,55 +267,78 @@ static CURLcode pre_transfer(struct GlobalConfig *global,
  struct_stat fileinfo;
  CURLcode result = CURLE_OK;
 
- if(per->uploadfile && !stdin_upload(per->uploadfile)) {
- /* VMS Note:
- *
- * Reading binary from files can be a problem... Only FIXED, VAR
- * etc WITHOUT implied CC will work. Others need a \n appended to
- * a line
- *
- * - Stat gives a size but this is UNRELIABLE in VMS. E.g.
- * a fixed file with implied CC needs to have a byte added for every
- * record processed, this can be derived from Filesize & recordsize
- * for VARiable record files the records need to be counted! for
- * every record add 1 for linefeed and subtract 2 for the record
- * header for VARIABLE header files only the bare record data needs
- * to be considered with one appended if implied CC
- */
+ if(per->uploadfile) {
+ bool isstdin = stdin_upload(per->uploadfile);
+ if(!isstdin) {
 #ifdef __VMS
- /* Calculate the real upload size for VMS */
- per->infd = -1;
- if(stat(per->uploadfile, &fileinfo) == 0) {
- fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo);
- switch(fileinfo.st_fab_rfm) {
- case FAB$C_VAR:
- case FAB$C_VFC:
- case FAB$C_STMCR:
- per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
- break;
- default:
- per->infd = open(per->uploadfile, O_RDONLY | O_BINARY,
- "rfm=stmlf", "ctx=stm");
+ /* VMS Note:
+ *
+ * Reading binary from files can be a problem... Only FIXED, VAR
+ * etc WITHOUT implied CC will work. Others need a \n appended to
+ * a line
+ *
+ * - Stat gives a size but this is UNRELIABLE in VMS. E.g. a
+ * fixed file with implied CC needs to have a byte added for every
+ * record processed, this can be derived from Filesize &
+ * recordsize for VARiable record files the records need to be
+ * counted! for every record add 1 for linefeed and subtract 2
+ * for the record header for VARIABLE header files only the bare
+ * record data needs to be considered with one appended if implied
+ * CC
+ */
+ /* Calculate the real upload size for VMS */
+ per->infd = -1;
+ if(stat(per->uploadfile, &fileinfo) == 0) {
+ fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo);
+ switch(fileinfo.st_fab_rfm) {
+ case FAB$C_VAR:
+ case FAB$C_VFC:
+ case FAB$C_STMCR:
+ per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
+ break;
+ default:
+ per->infd = open(per->uploadfile, O_RDONLY | O_BINARY,
+ "rfm=stmlf", "ctx=stm");
+ }
  }
+#else
+ per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
+#endif
  }
+
+#ifdef __VMS
  if(per->infd == -1)
 #else
- per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
- if((per->infd == -1) || fstat(per->infd, &fileinfo))
+ if(per->infd == -1 || fstat(per->infd, &fileinfo))
 #endif
  {
  helpf(tool_stderr, "Can't open '%s'", per->uploadfile);
- if(per->infd != -1) {
+ if(!isstdin && per->infd != -1) {
  close(per->infd);
  per->infd = STDIN_FILENO;
  }
  return CURLE_READ_ERROR;
  }
- per->infdopen = TRUE;
+ per->infdopen = !isstdin;
 
  /* we ignore file size for char/block devices, sockets, etc. */
  if(S_ISREG(fileinfo.st_mode))
+#ifdef __VMS
  uploadfilesize = fileinfo.st_size;
+#else
+ {
+ /* When the upload file is stdin, or when the upload file is
+ /dev/std{in,out,err} or /dev/fd/N on BSDs, the offset may not
+ be 0 */
+ off_t offset = lseek(per->infd, 0, SEEK_CUR);
+ if(offset >= 0)
+ uploadfilesize = fileinfo.st_size - offset;
+ else {
+ warnf(global, "Can't get file position of file descriptor %d ('%s')",
+ per->infd, per->uploadfile);
+ }
+ }
+#endif
 
 #ifdef DEBUGBUILD
  /* allow dedicated test cases to override */

tests/data/test1068

@@ -26,12 +26,12 @@ blablabla
 http
 </server>
 <name>
-HTTP PUT from stdin
+HTTP PUT from stdin pipe
 </name>
 <command>
 http://%HOSTIP:%HTTPPORT/bzz/%TESTNUMBER -T -
 </command>
-<stdin>
+<stdin pipe="yes">
 more than one byte
 </stdin>
 </client>

tests/data/test1069

@@ -17,12 +17,12 @@ HTTP/1.0
 http
 </server>
 <name>
-HTTP 1.0 PUT from stdin with no content length
+HTTP 1.0 PUT from stdin pipe with no content length
 </name>
 <command>
 http://%HOSTIP:%HTTPPORT/bzz/%TESTNUMBER -T - -0
 </command>
-<stdin>
+<stdin pipe="yes">
 this data can't be sent
 </stdin>
 </client>

tests/data/test1073

@@ -36,9 +36,9 @@ HTTP chunked PUT to HTTP 1.0 server with redirect
 <command>
 http://%HOSTIP:%HTTPPORT/%TESTNUMBER -T - -L
 </command>
-<stdin>
+<stdin pipe="yes">
 This is data we upload with PUT
-it comes from stdin so MUST be sent
+it comes from a pipe so MUST be sent
 with chunked encoding
 which is impossible in HTTP/1.0
 </stdin>
@@ -47,7 +47,7 @@ which is impossible in HTTP/1.0
 # Verify data after the test has been "shot"
 <verify>
 <errorcode>
-25
+65
 </errorcode>
 <protocol>
 PUT /%TESTNUMBER HTTP/1.1
@@ -58,12 +58,12 @@ Transfer-Encoding: chunked
 Expect: 100-continue
 
 %if hyper
-7A
+7B
 %else
-7a
+7b
 %endif
 This is data we upload with PUT
-it comes from stdin so MUST be sent
+it comes from a pipe so MUST be sent
 with chunked encoding
 which is impossible in HTTP/1.0
 

tests/data/test60

@@ -25,12 +25,12 @@ blablabla
 http
 </server>
 <name>
-HTTP PUT from stdin with wrong content-length
+HTTP PUT from stdin pipe with wrong content-length
 </name>
 <command>
 http://%HOSTIP:%HTTPPORT/bzz/%TESTNUMBER -T - -H "Content-Length: 1"
 </command>
-<stdin>
+<stdin pipe="yes">
 more than one byte
 </stdin>
 </client>