add support for requests with both MultipartFormDataItems and Content Providers (#1454)

* add support for requests with both MultipartFormDataItems and ContentProviders

* rework implementation

* use const auto & and fix offset calculation

* fix zero items

* snake case variables

* clang-format

* commonize get_multipart_content_provider, add Put() with MultipartFormDataProviderItems

* fix linker multiple definition error

* add test MultipartFormDataTest.DataProviderItems

httplib.h

@@ -369,6 +369,14 @@ using ContentProviderWithoutLength =
 
 using ContentProviderResourceReleaser = std::function<void(bool success)>;
 
+struct MultipartFormDataProvider {
+  std::string name;
+  ContentProviderWithoutLength provider;
+  std::string filename;
+  std::string content_type;
+};
+using MultipartFormDataProviderItems = std::vector<MultipartFormDataProvider>;
+
 using ContentReceiverWithProgress =
     std::function<bool(const char *data, size_t data_length, uint64_t offset,
                        uint64_t total_length)>;
@@ -934,6 +942,9 @@ public:
               const MultipartFormDataItems &items);
   Result Post(const std::string &path, const Headers &headers,
               const MultipartFormDataItems &items, const std::string &boundary);
+  Result Post(const std::string &path, const Headers &headers,
+              const MultipartFormDataItems &items,
+              const MultipartFormDataProviderItems &provider_items);
 
   Result Put(const std::string &path);
   Result Put(const std::string &path, const char *body, size_t content_length,
@@ -963,6 +974,9 @@ public:
              const MultipartFormDataItems &items);
   Result Put(const std::string &path, const Headers &headers,
              const MultipartFormDataItems &items, const std::string &boundary);
+  Result Put(const std::string &path, const Headers &headers,
+             const MultipartFormDataItems &items,
+             const MultipartFormDataProviderItems &provider_items);
 
   Result Patch(const std::string &path);
   Result Patch(const std::string &path, const char *body, size_t content_length,
@@ -1201,6 +1215,9 @@ private:
       ContentProvider content_provider,
       ContentProviderWithoutLength content_provider_without_length,
       const std::string &content_type);
+  ContentProviderWithoutLength get_multipart_content_provider(
+      const std::string &boundary, const MultipartFormDataItems &items,
+      const MultipartFormDataProviderItems &provider_items);
 
   std::string adjust_host_string(const std::string &host) const;
 
@@ -1296,6 +1313,10 @@ public:
               const MultipartFormDataItems &items);
   Result Post(const std::string &path, const Headers &headers,
               const MultipartFormDataItems &items, const std::string &boundary);
+  Result Post(const std::string &path, const Headers &headers,
+              const MultipartFormDataItems &items,
+              const MultipartFormDataProviderItems &provider_items);
+
   Result Put(const std::string &path);
   Result Put(const std::string &path, const char *body, size_t content_length,
              const std::string &content_type);
@@ -1324,6 +1345,10 @@ public:
              const MultipartFormDataItems &items);
   Result Put(const std::string &path, const Headers &headers,
              const MultipartFormDataItems &items, const std::string &boundary);
+  Result Put(const std::string &path, const Headers &headers,
+             const MultipartFormDataItems &items,
+             const MultipartFormDataProviderItems &provider_items);
+
   Result Patch(const std::string &path);
   Result Patch(const std::string &path, const char *body, size_t content_length,
                const std::string &content_type);
@@ -2854,8 +2879,7 @@ inline socket_t create_client_socket(
 }
 
 inline bool get_ip_and_port(const struct sockaddr_storage &addr,
-                            socklen_t addr_len, std::string &ip,
+                            socklen_t addr_len, std::string &ip, int &port) {
-                            int &port) {
   if (addr.ss_family == AF_INET) {
     port = ntohs(reinterpret_cast<const struct sockaddr_in *>(&addr)->sin_port);
   } else if (addr.ss_family == AF_INET6) {
@@ -4129,29 +4153,48 @@ inline bool is_multipart_boundary_chars_valid(const std::string &boundary) {
   return valid;
 }
 
+template <typename T>
+inline std::string
+serialize_multipart_formdata_item_begin(const T &item,
+                                        const std::string &boundary) {
+  std::string body = "--" + boundary + "\r\n";
+  body += "Content-Disposition: form-data; name=\"" + item.name + "\"";
+  if (!item.filename.empty()) {
+    body += "; filename=\"" + item.filename + "\"";
+  }
+  body += "\r\n";
+  if (!item.content_type.empty()) {
+    body += "Content-Type: " + item.content_type + "\r\n";
+  }
+  body += "\r\n";
+
+  return body;
+}
+
+inline std::string serialize_multipart_formdata_item_end() { return "\r\n"; }
+
+inline std::string
+serialize_multipart_formdata_finish(const std::string &boundary) {
+  return "--" + boundary + "--\r\n";
+}
+
+inline std::string
+serialize_multipart_formdata_get_content_type(const std::string &boundary) {
+  return "multipart/form-data; boundary=" + boundary;
+}
+
 inline std::string
 serialize_multipart_formdata(const MultipartFormDataItems &items,
-                             const std::string &boundary,
+                             const std::string &boundary, bool finish = true) {
-                             std::string &content_type) {
   std::string body;
 
   for (const auto &item : items) {
-    body += "--" + boundary + "\r\n";
+    body += serialize_multipart_formdata_item_begin(item, boundary);
-    body += "Content-Disposition: form-data; name=\"" + item.name + "\"";
+    body += item.content + serialize_multipart_formdata_item_end();
-    if (!item.filename.empty()) {
-      body += "; filename=\"" + item.filename + "\"";
-    }
-    body += "\r\n";
-    if (!item.content_type.empty()) {
-      body += "Content-Type: " + item.content_type + "\r\n";
-    }
-    body += "\r\n";
-    body += item.content + "\r\n";
   }
 
-  body += "--" + boundary + "--\r\n";
+  if (finish) body += serialize_multipart_formdata_finish(boundary);
 
-  content_type = "multipart/form-data; boundary=" + boundary;
   return body;
 }
 
@@ -4536,8 +4579,8 @@ inline void hosted_at(const std::string &hostname,
         *reinterpret_cast<struct sockaddr_storage *>(rp->ai_addr);
     std::string ip;
     int dummy = -1;
-    if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage),
+    if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip,
-                                ip, dummy)) {
+                                dummy)) {
       addrs.push_back(ip);
     }
   }
@@ -6647,6 +6690,49 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,
   return true;
 }
 
+inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(
+    const std::string &boundary, const MultipartFormDataItems &items,
+    const MultipartFormDataProviderItems &provider_items) {
+  size_t cur_item = 0, cur_start = 0;
+  // cur_item and cur_start are copied to within the std::function and maintain
+  // state between successive calls
+  return [&, cur_item, cur_start](size_t offset,
+                                  DataSink &sink) mutable -> bool {
+    if (!offset && items.size()) {
+      sink.os << detail::serialize_multipart_formdata(items, boundary, false);
+      return true;
+    } else if (cur_item < provider_items.size()) {
+      if (!cur_start) {
+        const auto &begin = detail::serialize_multipart_formdata_item_begin(
+            provider_items[cur_item], boundary);
+        offset += begin.size();
+        cur_start = offset;
+        sink.os << begin;
+      }
+
+      DataSink cur_sink;
+      bool has_data = true;
+      cur_sink.write = sink.write;
+      cur_sink.done = [&]() { has_data = false; };
+      cur_sink.is_writable = sink.is_writable;
+
+      if (!provider_items[cur_item].provider(offset - cur_start, cur_sink))
+        return false;
+
+      if (!has_data) {
+        sink.os << detail::serialize_multipart_formdata_item_end();
+        cur_item++;
+        cur_start = 0;
+      }
+      return true;
+    } else {
+      sink.os << detail::serialize_multipart_formdata_finish(boundary);
+      sink.done();
+      return true;
+    }
+  };
+}
+
 inline bool
 ClientImpl::process_socket(const Socket &socket,
                            std::function<bool(Stream &strm)> callback) {
@@ -6869,9 +6955,10 @@ inline Result ClientImpl::Post(const std::string &path,
 
 inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
                                const MultipartFormDataItems &items) {
-  std::string content_type;
+  const auto &boundary = detail::make_multipart_data_boundary();
-  const auto &body = detail::serialize_multipart_formdata(
+  const auto &content_type =
-      items, detail::make_multipart_data_boundary(), content_type);
+      detail::serialize_multipart_formdata_get_content_type(boundary);
+  const auto &body = detail::serialize_multipart_formdata(items, boundary);
   return Post(path, headers, body, content_type.c_str());
 }
 
@@ -6882,12 +6969,25 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
     return Result{nullptr, Error::UnsupportedMultipartBoundaryChars};
   }
 
-  std::string content_type;
+  const auto &content_type =
-  const auto &body =
+      detail::serialize_multipart_formdata_get_content_type(boundary);
-      detail::serialize_multipart_formdata(items, boundary, content_type);
+  const auto &body = detail::serialize_multipart_formdata(items, boundary);
   return Post(path, headers, body, content_type.c_str());
 }
 
+inline Result
+ClientImpl::Post(const std::string &path, const Headers &headers,
+                 const MultipartFormDataItems &items,
+                 const MultipartFormDataProviderItems &provider_items) {
+  const auto &boundary = detail::make_multipart_data_boundary();
+  const auto &content_type =
+      detail::serialize_multipart_formdata_get_content_type(boundary);
+  return send_with_content_provider(
+      "POST", path, headers, nullptr, 0, nullptr,
+      get_multipart_content_provider(boundary, items, provider_items),
+      content_type);
+}
+
 inline Result ClientImpl::Put(const std::string &path) {
   return Put(path, std::string(), std::string());
 }
@@ -6964,9 +7064,10 @@ inline Result ClientImpl::Put(const std::string &path,
 
 inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
                               const MultipartFormDataItems &items) {
-  std::string content_type;
+  const auto &boundary = detail::make_multipart_data_boundary();
-  const auto &body = detail::serialize_multipart_formdata(
+  const auto &content_type =
-      items, detail::make_multipart_data_boundary(), content_type);
+      detail::serialize_multipart_formdata_get_content_type(boundary);
+  const auto &body = detail::serialize_multipart_formdata(items, boundary);
   return Put(path, headers, body, content_type);
 }
 
@@ -6977,12 +7078,24 @@ inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
     return Result{nullptr, Error::UnsupportedMultipartBoundaryChars};
   }
 
-  std::string content_type;
+  const auto &content_type =
-  const auto &body =
+      detail::serialize_multipart_formdata_get_content_type(boundary);
-      detail::serialize_multipart_formdata(items, boundary, content_type);
+  const auto &body = detail::serialize_multipart_formdata(items, boundary);
   return Put(path, headers, body, content_type);
 }
 
+inline Result
+ClientImpl::Put(const std::string &path, const Headers &headers,
+                const MultipartFormDataItems &items,
+                const MultipartFormDataProviderItems &provider_items) {
+  const auto &boundary = detail::make_multipart_data_boundary();
+  const auto &content_type =
+      detail::serialize_multipart_formdata_get_content_type(boundary);
+  return send_with_content_provider(
+      "PUT", path, headers, nullptr, 0, nullptr,
+      get_multipart_content_provider(boundary, items, provider_items),
+      content_type);
+}
 inline Result ClientImpl::Patch(const std::string &path) {
   return Patch(path, std::string(), std::string());
 }
@@ -7443,7 +7556,7 @@ inline void SSLSocketStream::get_remote_ip_and_port(std::string &ip,
 }
 
 inline void SSLSocketStream::get_local_ip_and_port(std::string &ip,
-                                                    int &port) const {
+                                                   int &port) const {
   detail::get_local_ip_and_port(sock_, ip, port);
 }
 
@@ -8147,6 +8260,12 @@ inline Result Client::Post(const std::string &path, const Headers &headers,
                            const std::string &boundary) {
   return cli_->Post(path, headers, items, boundary);
 }
+inline Result
+Client::Post(const std::string &path, const Headers &headers,
+             const MultipartFormDataItems &items,
+             const MultipartFormDataProviderItems &provider_items) {
+  return cli_->Post(path, headers, items, provider_items);
+}
 inline Result Client::Put(const std::string &path) { return cli_->Put(path); }
 inline Result Client::Put(const std::string &path, const char *body,
                           size_t content_length,
@@ -8210,6 +8329,12 @@ inline Result Client::Put(const std::string &path, const Headers &headers,
                           const std::string &boundary) {
   return cli_->Put(path, headers, items, boundary);
 }
+inline Result
+Client::Put(const std::string &path, const Headers &headers,
+            const MultipartFormDataItems &items,
+            const MultipartFormDataProviderItems &provider_items) {
+  return cli_->Put(path, headers, items, provider_items);
+}
 inline Result Client::Patch(const std::string &path) {
   return cli_->Patch(path);
 }

test/test.cc

@@ -5146,6 +5146,204 @@ TEST(MultipartFormDataTest, LargeData) {
   t.join();
 }
 
+TEST(MultipartFormDataTest, DataProviderItems) {
+
+  std::random_device seed_gen;
+  std::mt19937 random(seed_gen());
+
+  std::string rand1;
+  rand1.resize(1000);
+  std::generate(rand1.begin(), rand1.end(), [&]() { return random(); });
+
+  std::string rand2;
+  rand2.resize(3000);
+  std::generate(rand2.begin(), rand2.end(), [&]() { return random(); });
+
+  SSLServer svr(SERVER_CERT_FILE, SERVER_PRIVATE_KEY_FILE);
+
+  svr.Post("/post-none", [&](const Request &req, Response & /*res*/,
+                             const ContentReader &content_reader) {
+    ASSERT_FALSE(req.is_multipart_form_data());
+
+    std::string body;
+    content_reader([&](const char *data, size_t data_length) {
+      body.append(data, data_length);
+      return true;
+    });
+
+    EXPECT_EQ(body, "");
+  });
+
+  svr.Post("/post-items", [&](const Request &req, Response & /*res*/,
+                              const ContentReader &content_reader) {
+    ASSERT_TRUE(req.is_multipart_form_data());
+    MultipartFormDataItems files;
+    content_reader(
+        [&](const MultipartFormData &file) {
+          files.push_back(file);
+          return true;
+        },
+        [&](const char *data, size_t data_length) {
+          files.back().content.append(data, data_length);
+          return true;
+        });
+
+    ASSERT_TRUE(files.size() == 2);
+
+    EXPECT_EQ(std::string(files[0].name), "name1");
+    EXPECT_EQ(files[0].content, "Testing123");
+    EXPECT_EQ(files[0].filename, "filename1");
+    EXPECT_EQ(files[0].content_type, "application/octet-stream");
+
+    EXPECT_EQ(files[1].name, "name2");
+    EXPECT_EQ(files[1].content, "Testing456");
+    EXPECT_EQ(files[1].filename, "");
+    EXPECT_EQ(files[1].content_type, "");
+  });
+
+  svr.Post("/post-providers", [&](const Request &req, Response & /*res*/,
+                                  const ContentReader &content_reader) {
+    ASSERT_TRUE(req.is_multipart_form_data());
+    MultipartFormDataItems files;
+    content_reader(
+        [&](const MultipartFormData &file) {
+          files.push_back(file);
+          return true;
+        },
+        [&](const char *data, size_t data_length) {
+          files.back().content.append(data, data_length);
+          return true;
+        });
+
+    ASSERT_TRUE(files.size() == 2);
+
+    EXPECT_EQ(files[0].name, "name3");
+    EXPECT_EQ(files[0].content, rand1);
+    EXPECT_EQ(files[0].filename, "filename3");
+    EXPECT_EQ(files[0].content_type, "");
+
+    EXPECT_EQ(files[1].name, "name4");
+    EXPECT_EQ(files[1].content, rand2);
+    EXPECT_EQ(files[1].filename, "filename4");
+    EXPECT_EQ(files[1].content_type, "");
+  });
+
+  svr.Post("/post-both", [&](const Request &req, Response & /*res*/,
+                             const ContentReader &content_reader) {
+    ASSERT_TRUE(req.is_multipart_form_data());
+    MultipartFormDataItems files;
+    content_reader(
+        [&](const MultipartFormData &file) {
+          files.push_back(file);
+          return true;
+        },
+        [&](const char *data, size_t data_length) {
+          files.back().content.append(data, data_length);
+          return true;
+        });
+
+    ASSERT_TRUE(files.size() == 4);
+
+    EXPECT_EQ(std::string(files[0].name), "name1");
+    EXPECT_EQ(files[0].content, "Testing123");
+    EXPECT_EQ(files[0].filename, "filename1");
+    EXPECT_EQ(files[0].content_type, "application/octet-stream");
+
+    EXPECT_EQ(files[1].name, "name2");
+    EXPECT_EQ(files[1].content, "Testing456");
+    EXPECT_EQ(files[1].filename, "");
+    EXPECT_EQ(files[1].content_type, "");
+
+    EXPECT_EQ(files[2].name, "name3");
+    EXPECT_EQ(files[2].content, rand1);
+    EXPECT_EQ(files[2].filename, "filename3");
+    EXPECT_EQ(files[2].content_type, "");
+
+    EXPECT_EQ(files[3].name, "name4");
+    EXPECT_EQ(files[3].content, rand2);
+    EXPECT_EQ(files[3].filename, "filename4");
+    EXPECT_EQ(files[3].content_type, "");
+  });
+
+  auto t = std::thread([&]() { svr.listen("localhost", 8080); });
+  while (!svr.is_running()) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+  }
+  std::this_thread::sleep_for(std::chrono::seconds(1));
+
+  {
+    Client cli("https://localhost:8080");
+    cli.enable_server_certificate_verification(false);
+
+    MultipartFormDataItems items{
+        {"name1", "Testing123", "filename1", "application/octet-stream"},
+        {"name2", "Testing456", "", ""}, // not a file
+    };
+
+    {
+      auto res = cli.Post("/post-none", {}, {}, {});
+      ASSERT_TRUE(res);
+      ASSERT_EQ(200, res->status);
+    }
+
+    MultipartFormDataProviderItems providers;
+
+    {
+      auto res =
+          cli.Post("/post-items", {}, items, providers); // empty providers
+      ASSERT_TRUE(res);
+      ASSERT_EQ(200, res->status);
+    }
+
+    providers.push_back({"name3",
+                         [&](size_t offset, httplib::DataSink &sink) -> bool {
+                           // test the offset is given correctly at each step
+                           if (!offset)
+                             sink.os.write(rand1.data(), 30);
+                           else if (offset == 30)
+                             sink.os.write(rand1.data() + 30, 300);
+                           else if (offset == 330)
+                             sink.os.write(rand1.data() + 330, 670);
+                           else if (offset == rand1.size())
+                             sink.done();
+                           return true;
+                         },
+                         "filename3",
+                         {}});
+
+    providers.push_back({"name4",
+                         [&](size_t offset, httplib::DataSink &sink) -> bool {
+                           // test the offset is given correctly at each step
+                           if (!offset)
+                             sink.os.write(rand2.data(), 2000);
+                           else if (offset == 2000)
+                             sink.os.write(rand2.data() + 2000, 1);
+                           else if (offset == 2001)
+                             sink.os.write(rand2.data() + 2001, 999);
+                           else if (offset == rand2.size())
+                             sink.done();
+                           return true;
+                         },
+                         "filename4",
+                         {}});
+
+    {
+      auto res = cli.Post("/post-providers", {}, {}, providers);
+      ASSERT_TRUE(res);
+      ASSERT_EQ(200, res->status);
+    }
+
+    {
+      auto res = cli.Post("/post-both", {}, items, providers);
+      ASSERT_TRUE(res);
+      ASSERT_EQ(200, res->status);
+    }
+  }
+
+  svr.stop();
+  t.join();
+}
+
 TEST(MultipartFormDataTest, WithPreamble) {
   Server svr;
   svr.Post("/post", [&](const Request & /*req*/, Response &res) {