Handle uploads > 4GiB

API-C.md

@@ -48,6 +48,8 @@ C Library for creating WebAssembly modules for use with NGINX Unit.
   * [luw_mem_writep](#luw_mem_writep)
   * [luw_mem_writep_data](#luw_mem_writep_data)
   * [luw_req_buf_append](#luw_req_buf_append)
+  * [luw_req_buf_copy](#luw_req_buf_copy)
+  * [luw_mem_splice_file](#luw_mem_splice_file)
   * [luw_mem_fill_buf_from_req](#luw_mem_fill_buf_from_req)
   * [luw_mem_reset](#luw_mem_reset)
   * [luw_http_set_response_status](#luw_http_set_response_status)
@@ -207,17 +209,19 @@ struct luw_req {
         u32 server_name_off;
         u32 server_name_len;
 
-        u32 content_off;
-        u32 content_len;
+        u64 content_len;
+        u64 total_content_sent;
         u32 content_sent;
-        u32 total_content_sent;
+        u32 content_off;
 
         u32 request_size;
 
         u32 nr_fields;
 
         u32 tls;
 
+        char __pad[4];
+
         struct luw_hdr_field fields[];
 };
 ```
@@ -701,11 +705,12 @@ This function returns a pointer to the start of the request body.
 ### luw_get_http_content_len
 
 ```C
-size_t luw_get_http_content_len(const luw_ctx_t *ctx);
+u64 luw_get_http_content_len(const luw_ctx_t *ctx);
 ```
 
 This function returns the size of the overall content. I.e Content-Length.
 
+Prior to version 0.3.0 it returned a size_t
 
 ### luw_get_http_content_sent
 
@@ -720,14 +725,14 @@ split over several calls to luw_request_handler().
 ### luw_get_http_total_content_sent
 
 ```C
-size_t luw_get_http_total_content_sent(const luw_ctx_t *ctx);
+u64 luw_get_http_total_content_sent(const luw_ctx_t *ctx);
 ```
 
 This function returns the total length of the content that was sent to the
 WebAssembly module so far. Remember, a single HTTP request may be split over
 several calls to luw_request_handler().
 
-_Version: 0.2.0_
+_Version: 0.2.0_ Prior to 0.3.0 it returned a size_t
 
 ### luw_http_is_tls
 
@@ -867,6 +872,93 @@ int luw_request_handler(u8 *addr)
 }
 ```
 
+### luw_req_buf_copy
+
+```C
+void luw_req_buf_copy(luw_ctx_t *ctx, const u8 *src);
+```
+
+This function is analogous to [luw_req_buf_append](#luw_req_buf_append) but
+rather than appending the request data contained in _src_ to the previously
+setup *request_buffer* with luw_set_req_buf(), it simply overwrites what's
+currently there.
+
+This function could be used to handle large requests/uploads that you want to
+save out to disk or some such and can't buffer it all in memory.
+
+Example
+
+```C
+int luw_request_handler(u8 *addr)
+{
+        const u8 *buf;
+        ssize_t bytes_wrote;
+
+        if (total_bytes_wrote == 0) {
+                luw_init_ctx(&ctx, addr, 0);
+                luw_set_req_buf(&ctx, &request_buf, LUW_SRB_NONE);
+
+                fd = open("/var/tmp/large-file.dat", O_CREAT|O_TRUNC|O_WRONLY,
+                          0666);
+        } else {
+                luw_req_buf_copy(&ctx, addr);
+        }
+
+        buf = luw_get_http_content(&ctx);
+        bytes_wrote = write(fd, buf, luw_get_http_content_sent(&ctx));
+        if (bytes_wrote == -1)
+                return -1;
+
+        total_bytes_wrote += bytes_wrote;
+        if (total_bytes_wrote == luw_get_http_content_len(&ctx))
+                luw_http_response_end();
+
+        return 0;
+}
+```
+
+_Version: 0.3.0_
+
+### luw_mem_splice_file
+
+```C
+ssize_t luw_mem_splice_file(const u8 *src, int fd);
+```
+
+This function write(2)'s the request data directly from the shared memory
+(_src_) to the file represented by the given file-descriptor (_fd_).
+
+This can be used as an alternative to [luw_req_buf_copy](#luw_req_buf_copy)
+and avoids an extra copying of the request data.
+
+Example
+
+```C
+int luw_request_handler(u8 *addr) {
+        ssize_t bytes_wrote;
+
+        if (total_bytes_wrote == 0) {
+                luw_init_ctx(&ctx, addr, 0);
+                luw_set_req_buf(&ctx, &request_buf, LUW_SRB_NONE);
+
+                fd = open("/var/tmp/large-file.dat", O_CREAT|O_TRUNC|O_WRONLY,
+                          0666);
+        }
+
+        bytes_wrote = luw_mem_splice_file(addr, fd);
+        if (bytes_wrote == -1)
+                return -1;
+
+        total_bytes_wrote += bytes_wrote;
+        if (total_bytes_wrote == luw_get_http_content_len(&ctx))
+                luw_http_response_end();
+
+        return 0;
+}
+```
+
+_Version: 0.3.0_
+
 ### luw_mem_fill_buf_from_req
 
 ```C

API-Rust.md

@@ -76,6 +76,8 @@ and there isn't a real need to create wrappers specifically for them.
   * [uwr_get_response_data_size](#uwr_get_response_data_size)
   * [uwr_mem_write_buf](#uwr_mem_write_buf)
   * [uwr_req_buf_append](#uwr_req_buf_append)
+  * [uwr_req_buf_copy](#uwr_req_buf_copy)
+  * [uwr_mem_splice_file](#uwr_mem_splice_file)
   * [uwr_mem_fill_buf_from_req](#uwr_mem_fill_buf_from_req)
   * [uwr_mem_reset](#uwr_mem_reset)
   * [uwr_http_set_response_status](#uwr_http_set_response_status)
@@ -667,11 +669,12 @@ _Version: 0.2.0_
 ### uwr_get_http_content_len
 
 ```Rust
-pub fn uwr_get_http_content_len(ctx: *const luw_ctx_t) -> usize;
+pub fn uwr_get_http_content_len(ctx: *const luw_ctx_t) -> u64;
 ```
 
 This function returns the size of the overall content. I.e Content-Length.
 
+Prior to version 0.3.0 it returned a usize
 
 ### uwr_get_http_content_sent
 
@@ -686,14 +689,14 @@ split over several calls to luw_request_handler().
 ### uwr_get_http_total_content_sent
 
 ```Rust
-pub fn uwr_get_http_total_content_sent(ctx: *const luw_ctx_t) -> usize;
+pub fn uwr_get_http_total_content_sent(ctx: *const luw_ctx_t) -> u64;
 ```
 
 This function returns the total length of the content that was sent to the
 WebAssembly module so far. Remember, a single HTTP request may be split over
 several calls to luw_request_handler().
 
-_Version: 0.2.0_
+_Version: 0.2.0_ Prior to 0.3.0 it returned a usize
 
 ### uwr_http_is_tls
 
@@ -836,6 +839,70 @@ pub extern "C" fn uwr_request_handler(addr: *mut u8) -> i32 {
 }
 ```
 
+### uwr_req_buf_copy
+
+```Rust
+pub fn uwr_req_buf_copy(ctx: *mut luw_ctx_t, src: *const u8);
+```
+
+This function is analogous to [uwr_req_buf_append](#uwr_req_buf_append) but
+rather than appending the request data contained in _src_ to the previously
+setup *request_buffer* with uwr_set_req_buf(), it simply overwrites what's
+currently there.
+
+This function could be used to handle large requests/uploads that you want to
+save out to disk or some such and can't buffer it all in memory.
+
+### uwr_mem_splice_file
+
+```Rust
+pub fn uwr_mem_splice_file(src: *const u8, f: &mut File) -> isize;
+```
+This function write(2)'s the request data directly from the shared memory
+(_src_) to the file represented by the given _File_ object (_f_).
+
+This can be used as an alternative to [uwr_req_buf_copy](#uwr_req_buf_copy)
+and avoids an extra copying of the request data.
+
+Example
+
+```Rust
+pub extern "C" fn uwr_request_handler(addr: *mut u8) -> i32 {
+    let ctx: *mut luw_ctx_t = unsafe { &mut CTX };
+    let mut f;
+    let bytes_wrote: isize;
+    let mut total = unsafe { TOTAL_BYTES_WROTE };
+
+    if total == 0 {
+        uwr_init_ctx(ctx, addr, 0);
+        uwr_set_req_buf(ctx, unsafe { &mut REQUEST_BUF }, LUW_SRB_NONE);
+
+        f = File::create("/var/tmp/large-file.dat").unwrap();
+    } else {
+        f = File::options()
+            .append(true)
+            .open("/var/tmp/large-file.dat")
+            .unwrap();
+    }
+
+    bytes_wrote = uwr_mem_splice_file(addr, &mut f);
+    if bytes_wrote == -1 {
+        return -1;
+    }
+
+    total += bytes_wrote as u64;
+    if total == uwr_get_http_content_len(ctx) {
+        total = 0;
+
+        uwr_http_response_end();
+    }
+
+    unsafe { TOTAL_BYTES_WROTE = total };
+
+    return 0;
+}
+```
+
 ### uwr_mem_fill_buf_from_req
 
 ```Rust

README.md

@@ -264,7 +264,7 @@ repository root for more details) but will instead assume you already have a
 Unit with the WebAssembly language module already running, perhaps installed
 via a package.
 
-Create the following Unit config
+Create the following Unit config (editing the module paths as appropriate)
 
 ```JSON
 {
@@ -276,7 +276,7 @@ Create the following Unit config
 
     "settings": {
         "http": {
-            "max_body_size": 1073741824
+            "max_body_size": 8589934592
         }
     },
 
@@ -297,6 +297,14 @@ Create the following Unit config
                 "pass": "applications/luw-upload-reflector"
             }
         },
+        {
+            "match": {
+                "uri": "/large-upload*"
+            },
+            "action": {
+                "pass": "applications/large-upload"
+            }
+        },
         {
             "match": {
                 "uri": "/rust-echo*"
@@ -315,7 +323,15 @@ Create the following Unit config
         },
         {
             "match": {
-                "uri": "/hello-world*"
+                "uri": "/rust-large-upload*"
+            },
+            "action": {
+                "pass": "applications/rust-large-upload"
+            }
+        },
+        {
+            "match": {
+                "uri": "/rust-hello-world*"
             },
             "action": {
                 "pass": "applications/rust-hello-world"
@@ -342,6 +358,21 @@ Create the following Unit config
             "request_end_handler": "luw_request_end_handler",
             "response_end_handler": "luw_response_end_handler"
         },
+        "large-upload": {
+            "type": "wasm",
+            "module": "/path/to/unit-wasm/examples/c/large-upload.wasm",
+            "request_handler": "luw_request_handler",
+            "malloc_handler": "luw_malloc_handler",
+            "free_handler": "luw_free_handler",
+            "module_init_handler": "luw_module_init_handler",
+            "module_end_handler": "luw_module_end_handler",
+            "response_end_handler": "luw_response_end_handler",
+            "access": {
+                "filesystem": [
+                    "/var/tmp"
+                ]
+            }
+        },
         "rust-echo-request": {
             "type": "wasm",
             "module": "/path/to/unit-wasm/examples/rust/echo-request/target/wasm32-wasi/debug/rust_echo_request.wasm",
@@ -360,6 +391,21 @@ Create the following Unit config
             "request_end_handler": "uwr_request_end_handler",
             "response_end_handler": "uwr_response_end_handler"
         },
+        "rust-large-upload": {
+            "type": "wasm",
+            "module": "/path/to/src/unit-wasm/examples/rust/large-upload/target/wasm32-wasi/debug/rust_large_upload.wasm",
+            "request_handler": "uwr_request_handler",
+            "malloc_handler": "luw_malloc_handler",
+            "free_handler": "luw_free_handler",
+            "module_init_handler": "uwr_module_init_handler",
+            "module_end_handler": "uwr_module_end_handler",
+            "response_end_handler": "uwr_response_end_handler",
+            "access": {
+                "filesystem": [
+                    "/var/tmp"
+                ]
+            }
+        },
         "rust-hello-world": {
             "type": "wasm",
             "module": "/path/to/unit-wasm/examples/rust/hello-world/target/wasm32-wasi/debug/rust_hello_world.wasm",

examples/c/Makefile

@@ -12,7 +12,9 @@ luw_deps = $(LUW_SRCDIR)/libunit-wasm.a \
 
 examples: examples-luw
 
-examples-luw: luw-echo-request.wasm luw-upload-reflector.wasm
+examples-luw: luw-echo-request.wasm \
+	      luw-upload-reflector.wasm \
+	      large-upload.wasm
 
 examples-raw: echo-request-raw.wasm upload-reflector-raw.wasm
 
@@ -36,5 +38,9 @@ upload-reflector-raw.wasm: upload-reflector-raw.c unit-wasm-raw.o
 	$(PP_CCLNK) $(SDIR)/$@
 	$(v)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $< unit-wasm-raw.o
 
+large-upload.wasm: large-upload.c $(luw_deps)
+	$(PP_CCLNK) $(SDIR)/$@
+	$(v)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $< $(LIBS)
+
 clean:
 	rm -f *.wasm *.o *.gch