curl: add byte range support to --variable reading from file

Allowing --variable read a portion of provided files, makes curl work on partial files for any options that accepts strings. Like --data and others. The byte offset is provided within brackets, with a semicolon separator like: --variable name@file;[100-200]" Inspired by #14479 Assisted-by: Manuel Einfalt Test 784 - 789. Documentation update provided. Closes #15739
2025-02-23 15:10:03 +08:00 · 2024-12-21 11:46:27 +01:00 · 2024-12-21 11:46:27 +01:00 · 40c264db61
commit 40c264db61
parent fc3e1cbc50
13 changed files with 453 additions and 11 deletions
--- a/docs/cmdline-opts/variable.md
+++ b/docs/cmdline-opts/variable.md
@ -36,6 +36,15 @@ the environment variable is not set, use --variable %name=content or
 --variable %name@content. Note that on some systems - but not all -
 environment variables are case insensitive.

+Added in curl 8.12.0: when getting contents from a file, you can request to
+get a byte range from it by appending ";[start-end]" to the filename, where
+*start* and *end* are byte offsets to include from the file. For example,
+asking for offset "2-10" means offset two to offset ten, including the byte
+offset 10, meaning 9 bytes in total. "2-2" means a single byte at offset 2.
+Not providing a second number implies to the end of the file. The start offset
+cannot be larger than the end offset. Asking for a range that is outside of
+the file size makes the variable contents empty.
+
 To assign a variable using contents from another variable, use
 --expand-variable. Like for example assigning a new variable using contents
 from two other:
--- a/src/tool_getparam.h
+++ b/src/tool_getparam.h
@ -348,6 +348,7 @@ typedef enum {
  PARAM_READ_ERROR,
  PARAM_EXPAND_ERROR, /* --expand problem */
  PARAM_BLANK_STRING,
+  PARAM_VAR_SYNTAX, /* --variable syntax error */
  PARAM_LAST
 } ParameterError;

--- a/src/tool_helpers.c
+++ b/src/tool_helpers.c
@ -75,6 +75,8 @@ const char *param2text(ParameterError error)
    return "variable expansion failure";
  case PARAM_BLANK_STRING:
    return "blank argument where content is expected";
+  case PARAM_VAR_SYNTAX:
+    return "syntax error in --variable argument";
  default:
    return "unknown error";
  }
--- a/src/tool_paramhlp.c
+++ b/src/tool_paramhlp.c
@ -124,15 +124,45 @@ ParameterError file2string(char **bufp, FILE *file)
  return PARAM_OK;
 }

-ParameterError file2memory(char **bufp, size_t *size, FILE *file)
+static int myfseek(void *stream, curl_off_t offset, int whence)
+{
+#if defined(_WIN32) && defined(USE_WIN32_LARGE_FILES)
+  return _fseeki64(stream, (__int64)offset, whence);
+#elif defined(HAVE_FSEEKO) && defined(HAVE_DECL_FSEEKO)
+  return fseeko(stream, (off_t)offset, whence);
+#else
+  if(offset > LONG_MAX)
+    return -1;
+  return fseek(stream, (long)offset, whence);
+#endif
+}
+
+ParameterError file2memory_range(char **bufp, size_t *size, FILE *file,
+                                 curl_off_t starto, curl_off_t endo)
 {
  if(file) {
    size_t nread;
    struct curlx_dynbuf dyn;
+    curl_off_t offset = 0;
+    curl_off_t throwaway = 0;
+
+    if(starto) {
+      if(file != stdin) {
+        if(myfseek(file, starto, SEEK_SET))
+          return PARAM_READ_ERROR;
+        offset = starto;
+      }
+      else
+        /* we can't seek stdin, read 'starto' bytes and throw them away */
+        throwaway = starto;
+    }
+
    /* The size needs to fit in an int later */
    curlx_dyn_init(&dyn, MAX_FILE2MEMORY);
    do {
      char buffer[4096];
+      size_t n_add;
+      char *ptr_add;
      nread = fread(buffer, 1, sizeof(buffer), file);
      if(ferror(file)) {
        curlx_dyn_free(&dyn);
@ -140,9 +170,35 @@ ParameterError file2memory(char **bufp, size_t *size, FILE *file)
        *bufp = NULL;
        return PARAM_READ_ERROR;
      }
-      if(nread)
-        if(curlx_dyn_addn(&dyn, buffer, nread))
-          return PARAM_NO_MEM;
+      n_add = nread;
+      ptr_add = buffer;
+      if(nread) {
+        if(throwaway) {
+          if(throwaway >= (curl_off_t)nread) {
+            throwaway -= nread;
+            offset += nread;
+            n_add = 0; /* nothing to add */
+          }
+          else {
+            /* append the trailing piece */
+            n_add = (size_t)(nread - throwaway);
+            ptr_add = &buffer[throwaway];
+            offset += throwaway;
+            throwaway = 0;
+          }
+        }
+        if(n_add) {
+          if((curl_off_t)(n_add + offset) > endo)
+            n_add = (size_t)(endo - offset + 1);
+
+          if(curlx_dyn_addn(&dyn, ptr_add, n_add))
+            return PARAM_NO_MEM;
+
+          offset += n_add;
+          if(offset > endo)
+            break;
+        }
+      }
    } while(!feof(file));
    *size = curlx_dyn_len(&dyn);
    *bufp = curlx_dyn_ptr(&dyn);
@ -154,6 +210,11 @@ ParameterError file2memory(char **bufp, size_t *size, FILE *file)
  return PARAM_OK;
 }

+ParameterError file2memory(char **bufp, size_t *size, FILE *file)
+{
+  return file2memory_range(bufp, size, file, 0, CURL_OFF_T_MAX);
+}
+
 /*
 * Parse the string and write the long in the given address. Return PARAM_OK
 * on success, otherwise a parameter specific error enum.
--- a/src/tool_paramhlp.h
+++ b/src/tool_paramhlp.h
@ -37,6 +37,8 @@ ParameterError file2string(char **bufp, FILE *file);
 #endif

 ParameterError file2memory(char **bufp, size_t *size, FILE *file);
+ParameterError file2memory_range(char **bufp, size_t *size, FILE *file,
+                                 curl_off_t starto, curl_off_t endo);

 ParameterError str2num(long *val, const char *str);
 ParameterError str2unum(long *val, const char *str);
--- a/src/var.c
+++ b/src/var.c
@ -374,6 +374,8 @@ static ParameterError addvariable(struct GlobalConfig *global,
  return PARAM_NO_MEM;
 }

+#define MAX_FILENAME 10000
+
 ParameterError setvariable(struct GlobalConfig *global,
                           const char *input)
 {
@ -427,21 +429,56 @@ ParameterError setvariable(struct GlobalConfig *global,
    /* read from file or stdin */
    FILE *file;
    bool use_stdin;
+    char *range;
+    struct dynbuf fname;
+    curl_off_t startoffset = 0;
+    curl_off_t endoffset = CURL_OFF_T_MAX;
    line++;
+
+    Curl_dyn_init(&fname, MAX_FILENAME);
+
+    /* is there a byte range specified? ;[num-num] */
+    range = strstr(line, ";[");
+    if(range && ISDIGIT(range[2])) {
+      char *p = range;
+      char *endp;
+      if(curlx_strtoofft(&p[2], &endp, 10, &startoffset) || (*endp != '-'))
+        return PARAM_VAR_SYNTAX;
+      else {
+        p = endp + 1; /* pass the '-' */
+        if(*p != ']') {
+          if(curlx_strtoofft(p, &endp, 10, &endoffset) || (*endp != ']'))
+            return PARAM_VAR_SYNTAX;
+        }
+      }
+      if(startoffset > endoffset)
+        return PARAM_VAR_SYNTAX;
+      /* create a dynbuf for the filename without the range */
+      if(Curl_dyn_addn(&fname, line, (range - line)))
+        return PARAM_NO_MEM;
+      /* point to the new file name buffer */
+      line = Curl_dyn_ptr(&fname);
+    }
+
    use_stdin = !strcmp(line, "-");
    if(use_stdin)
      file = stdin;
    else {
      file = fopen(line, "rb");
      if(!file) {
-        errorf(global, "Failed to open %s", line);
-        return PARAM_READ_ERROR;
+        errorf(global, "Failed to open %s: %s", line,
+               strerror(errno));
+        err = PARAM_READ_ERROR;
      }
    }
-    err = file2memory(&content, &clen, file);
-    /* in case of out of memory, this should fail the entire operation */
-    contalloc = TRUE;
-    if(!use_stdin)
+    if(!err) {
+      err = file2memory_range(&content, &clen, file, startoffset, endoffset);
+      /* in case of out of memory, this should fail the entire operation */
+      if(clen)
+        contalloc = TRUE;
+    }
+    Curl_dyn_free(&fname);
+    if(!use_stdin && file)
      fclose(file);
    if(err)
      return err;
--- a/tests/data/Makefile.am
+++ b/tests/data/Makefile.am
@ -109,7 +109,8 @@ test718 test719 test720 test721 test722 test723 test724 test725 test726 \
 test727 test728 test729 test730 test731 test732 test733 test734 test735 \
 test736 test737 test738 test739 test740 test741 test742 \
 \
-test780 test781 test782 test783 \
+test780 test781 test782 test783 test784 test785 test786 test787 test788 \
+test789 \
 \
 test799 test800 test801 test802 test803 test804 test805 test806 test807 \
 test808 test809 test810 test811 test812 test813 test814 test815 test816 \
--- a/tests/data/test784
+++ b/tests/data/test784
@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file byte range
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[5-15]" --expand-data '{{name}}'
+</command>
+<file name="%LOGDIR/in%TESTNUMBER">
+On the first Monday of the month of April, 1625, the market town of Meung
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes" nonewline="yes">
+POST /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+Content-Length: 11
+Content-Type: application/x-www-form-urlencoded
+
+e first Mon
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test785
+++ b/tests/data/test785
@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file byte range without end
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[5-]" --expand-data '{{name}}'
+</command>
+<file name="%LOGDIR/in%TESTNUMBER">
+On the first Monday of the month of April, 1625, the market town of Meung
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+POST /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+Content-Length: 69
+Content-Type: application/x-www-form-urlencoded
+
+e first Monday of the month of April, 1625, the market town of Meung
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test786
+++ b/tests/data/test786
@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file byte range, reading from stdin
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@-;[5-15]" --expand-data '{{name}}'
+</command>
+<stdin>
+On the first Monday of the month of April, 1625, the market town of Meung
+</stdin>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol nonewline="yes">
+POST /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+Content-Length: 11
+Content-Type: application/x-www-form-urlencoded
+
+e first Mon
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test787
+++ b/tests/data/test787
@ -0,0 +1,35 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file byte range, bad range
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@&LOGDIR/fooo;[15-14]" --expand-data '{{name}}'
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<errorcode>
+2
+</errorcode>
+</verify>
+</testcase>
--- a/tests/data/test788
+++ b/tests/data/test788
@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file and single-byte byte range
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[15-15]" --expand-data '{{name}}'
+</command>
+<file name="%LOGDIR/in%TESTNUMBER">
+On the first Monday of the month of April, 1625, the market town of Meung
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes" nonewline="yes">
+POST /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+Content-Length: 1
+Content-Type: application/x-www-form-urlencoded
+
+n
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test789
+++ b/tests/data/test789
@ -0,0 +1,58 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file and byte range out of file
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[75-85]" --expand-data '{{name}}'
+</command>
+<file name="%LOGDIR/in%TESTNUMBER">
+On the first Monday of the month of April, 1625, the market town of Meung
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+POST /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+Content-Length: 0
+Content-Type: application/x-www-form-urlencoded
+
+</protocol>
+</verify>
+</testcase>