mirror/oatpp
2
0
Fork 0
mirror of https://github.com/oatpp/oatpp.git synced 2025-03-13 18:06:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

multipart data. streaming parser POC. WIP

Browse Source
This commit is contained in:
lganzzzo 2019-07-11 00:36:52 +03:00
parent f0fc31b555
commit d657d4f736
9 changed files with 515 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/CMakeLists.txt
View File

@ -141,6 +141,8 @@ add_library(oatpp
oatpp/web/client/HttpRequestExecutor.hpp
oatpp/web/client/RequestExecutor.cpp
oatpp/web/client/RequestExecutor.hpp
oatpp/web/mime/multipart/StatefulParser.cpp
oatpp/web/mime/multipart/StatefulParser.hpp
oatpp/web/protocol/CommunicationError.cpp
oatpp/web/protocol/CommunicationError.hpp
oatpp/web/protocol/http/Http.cpp

6
src/oatpp/web/client/ApiClient.hpp
View File

@ -22,8 +22,8 @@
*
***************************************************************************/
#ifndef glanzzzo_web_client_Client_hpp
#define glanzzzo_web_client_Client_hpp
#ifndef oatpp_web_client_Client_hpp
#define oatpp_web_client_Client_hpp
#include "./RequestExecutor.hpp"
@ -214,4 +214,4 @@ public:
}}}
#endif /* glanzzzo_web_client_Client_hpp */
#endif /* oatpp_web_client_Client_hpp */

268
src/oatpp/web/mime/multipart/StatefulParser.cpp Normal file
View File

@ -0,0 +1,268 @@
/***************************************************************************
*
* Project _____ __ ____ _ _
* ( _ ) /__\ (_ _)_| |_ _| |_
* )(_)( /(__)\ )( (_ _)(_ _)
* (_____)(__)(__)(__) |_| |_|
*
*
* Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
***************************************************************************/
#include "StatefulParser.hpp"
#include "oatpp/web/protocol/http/Http.hpp"
#include "oatpp/core/parser/Caret.hpp"
namespace oatpp { namespace web { namespace mime { namespace multipart {
void StatefulParser::onPartHeaders(const Headers& partHeaders) {
m_currPartIndex ++;
auto it = partHeaders.find("Content-Disposition");
if(it != partHeaders.end()) {
parser::Caret caret(it->second.toString());
if(caret.findText((p_char8)"name=", 5)) {
caret.inc(5);
parser::Caret::Label nameLabel(nullptr);
if(caret.isAtChar('"')) {
nameLabel = caret.parseStringEnclosed('"', '"', '\\');
} else if(caret.isAtChar('\'')) {
nameLabel = caret.parseStringEnclosed('\'', '\'', '\\');
} else {
nameLabel = caret.putLabel();
caret.findCharFromSet(" \t\n\r\f");
nameLabel.end();
}
if(nameLabel) {
m_currPartName = nameLabel.toString();
OATPP_LOGD("Part", "name='%s'", m_currPartName->getData());
for(auto& pair : partHeaders) {
auto key = pair.first.toString();
auto value = pair.second.toString();
OATPP_LOGD("header", "key='%s', value='%s'", key->getData(), value->getData());
}
} else {
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::onPartHeaders()]: Error. Can't parse part name.");
}
} else {
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::onPartHeaders()]: Error. Part name is missing.");
}
} else {
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::onPartHeaders()]: Error. Missing 'Content-Disposition' header.");
}
}
void StatefulParser::onPartData(p_char8 data, v_int32 size) {
oatpp::String text((const char*)data, size, true);
OATPP_LOGD("data", "part='%s', data='%s'", m_currPartName->getData(), text->getData());
}
v_int32 StatefulParser::parseNext_Boundary(p_char8 data, v_int32 size) {
p_char8 sampleData = m_nextBoundarySample->getData();
v_int32 sampleSize = m_nextBoundarySample->getSize();
if (m_currPartIndex == 0) {
sampleData = m_firstBoundarySample->getData();
sampleSize = m_firstBoundarySample->getSize();
} else {
sampleData = m_nextBoundarySample->getData();
sampleSize = m_nextBoundarySample->getSize();
}
v_int32 checkSize = sampleSize - m_currBoundaryCharIndex;
if(checkSize > size) {
checkSize = size;
}
parser::Caret caret(data, size);
if(caret.isAtText(&sampleData[m_currBoundaryCharIndex], checkSize, true)) {
m_currBoundaryCharIndex += caret.getPosition();
if(m_currBoundaryCharIndex == sampleSize) {
m_state = STATE_AFTER_BOUNDARY;
m_currBoundaryCharIndex = 0;
m_readingBody = false;
}
return caret.getPosition();
} else if(m_readingBody) {
if(m_currBoundaryCharIndex > 0) {
onPartData(sampleData, m_currBoundaryCharIndex);
}
m_state = STATE_DATA;
m_currBoundaryCharIndex = 0;
m_checkForBoundary = false;
return 0;
}
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::parseNext_Boundary()]: Error. Invalid state.");
}
v_int32 StatefulParser::parseNext_AfterBoundary(p_char8 data, v_int32 size) {
if(m_currBoundaryCharIndex == 0) {
if(data[0] == '-') {
m_finishingBoundary = true;
} else if(data[0] != '\r') {
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::parseNext_AfterBoundary()]: Error. Invalid char.");
}
}
if(size > 1 || m_currBoundaryCharIndex == 1) {
if (m_finishingBoundary && data[1 - m_currBoundaryCharIndex] == '-') {
m_state = STATE_DONE;
m_currBoundaryCharIndex = 0;
return 2 - m_currBoundaryCharIndex;
} else if (!m_finishingBoundary && data[1 - m_currBoundaryCharIndex] == '\n') {
m_state = STATE_HEADERS;
m_currBoundaryCharIndex = 0;
m_headerSectionEndAccumulator = 0;
return 2 - m_currBoundaryCharIndex;
} else {
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::parseNext_AfterBoundary()]: Error. Invalid trailing char.");
}
}
m_currBoundaryCharIndex = 1;
return 1;
}
v_int32 StatefulParser::parseNext_Headers(p_char8 data, v_int32 size) {
for(v_int32 i = 0; i < size; i ++) {
m_headerSectionEndAccumulator <<= 8;
m_headerSectionEndAccumulator |= data[i];
if(m_headerSectionEndAccumulator == HEADERS_SECTION_END) {
m_headersBuffer.write(data, i);
auto headersText = m_headersBuffer.toString();
m_headersBuffer.clear();
protocol::http::Status status;
parser::Caret caret(headersText);
Headers headers;
protocol::http::Parser::parseHeaders(headers, headersText.getPtr(), caret, status);
onPartHeaders(headers);
m_state = STATE_DATA;
m_checkForBoundary = true;
return i + 1;
}
}
m_headersBuffer.write(data, size);
return size;
}
v_int32 StatefulParser::parseNext_Data(p_char8 data, v_int32 size) {
parser::Caret caret(data, size);
bool rFound = caret.findChar('\r');
if(rFound && !m_checkForBoundary) {
caret.inc();
rFound = caret.findChar('\r');
}
m_checkForBoundary = true;
if(rFound) {
if(caret.getPosition() > 0) {
onPartData(data, caret.getPosition());
}
m_state = STATE_BOUNDARY;
m_readingBody = true;
return caret.getPosition();
} else {
onPartData(data, size);
}
return size;
}
v_int32 StatefulParser::parseNext(p_char8 data, v_int32 size) {
v_int32 pos = 0;
while(pos < size) {
switch (m_state) {
case STATE_BOUNDARY:
pos += parseNext_Boundary(&data[pos], size - pos);
break;
case STATE_AFTER_BOUNDARY:
pos += parseNext_AfterBoundary(&data[pos], size - pos);
break;
case STATE_HEADERS:
pos += parseNext_Headers(&data[pos], size - pos);
break;
case STATE_DATA:
pos += parseNext_Data(&data[pos], size - pos);
break;
case STATE_DONE:
return pos;
default:
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::parseNext()]: Error. Invalid state.");
}
}
return pos;
}
}}}}

112
src/oatpp/web/mime/multipart/StatefulParser.hpp Normal file
View File

@ -0,0 +1,112 @@
/***************************************************************************
*
* Project _____ __ ____ _ _
* ( _ ) /__\ (_ _)_| |_ _| |_
* )(_)( /(__)\ )( (_ _)(_ _)
* (_____)(__)(__)(__) |_| |_|
*
*
* Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
***************************************************************************/
#ifndef oatpp_web_mime_multipart_StatefulParser_hpp
#define oatpp_web_mime_multipart_StatefulParser_hpp
#include "oatpp/core/data/stream/ChunkedBuffer.hpp"
#include "oatpp/core/data/share/MemoryLabel.hpp"
#include "oatpp/core/Types.hpp"
#include <unordered_map>
namespace oatpp { namespace web { namespace mime { namespace multipart {
class StatefulParser {
private:
static constexpr v_int32 STATE_BOUNDARY = 0;
static constexpr v_int32 STATE_AFTER_BOUNDARY = 1;
static constexpr v_int32 STATE_HEADERS = 2;
static constexpr v_int32 STATE_DATA = 3;
static constexpr v_int32 STATE_DONE = 4;
private:
static constexpr v_int32 HEADERS_SECTION_END = ('\r' << 24) | ('\n' << 16) | ('\r' << 8) | ('\n');
public:
/**
* Typedef for headers map. Headers map key is case-insensitive.
* `std::unordered_map` of &id:oatpp::data::share::StringKeyLabelCI_FAST; and &id:oatpp::data::share::StringKeyLabel;.
*/
typedef std::unordered_map<oatpp::data::share::StringKeyLabelCI_FAST, oatpp::data::share::StringKeyLabel> Headers;
public:
class Listener {
public:
virtual void onPartHeaders(const oatpp::String& name, const Headers& partHeaders) = 0;
virtual void onPartData(const oatpp::String& name, p_char8 data, oatpp::data::v_io_size size) = 0;
};
private:
v_int32 m_state;
v_int32 m_currPartIndex;
v_int32 m_currBoundaryCharIndex;
bool m_checkForBoundary;
bool m_finishingBoundary;
bool m_readingBody;
v_word32 m_headerSectionEndAccumulator;
oatpp::String m_firstBoundarySample;
oatpp::String m_nextBoundarySample;
oatpp::String m_currPartName;
/*
* Headers of the part are stored in the buffer and are parsed as one chunk.
*/
oatpp::data::stream::ChunkedBuffer m_headersBuffer;
private:
void onPartHeaders(const Headers& partHeaders);
void onPartData(p_char8 data, v_int32 size);
private:
v_int32 parseNext_Boundary(p_char8 data, v_int32 size);
v_int32 parseNext_AfterBoundary(p_char8 data, v_int32 size);
v_int32 parseNext_Headers(p_char8 data, v_int32 size);
v_int32 parseNext_Data(p_char8 data, v_int32 size);
public:
StatefulParser(const oatpp::String& boundary)
: m_state(STATE_BOUNDARY)
, m_currPartIndex(0)
, m_currBoundaryCharIndex(0)
, m_checkForBoundary(true)
, m_finishingBoundary(false)
, m_readingBody(false)
, m_headerSectionEndAccumulator(0)
, m_firstBoundarySample("--" + boundary)
, m_nextBoundarySample("\r\n--" + boundary)
{}
v_int32 parseNext(p_char8 data, v_int32 size);
};
}}}}
#endif // oatpp_web_mime_multipart_StatefulParser_hpp

5
src/oatpp/web/protocol/http/incoming/RequestHeadersReader.cpp
View File

@ -31,8 +31,7 @@ namespace oatpp { namespace web { namespace protocol { namespace http { namespac
data::v_io_size RequestHeadersReader::readHeadersSection(const std::shared_ptr<oatpp::data::stream::IOStream>& connection,
oatpp::data::stream::OutputStream* bufferStream,
Result& result) {
v_word32 sectionEnd = ('\r' << 24) | ('\n' << 16) | ('\r' << 8) | ('\n');
v_word32 accumulator = 0;
v_int32 progress = 0;
data::v_io_size res;
@ -53,7 +52,7 @@ data::v_io_size RequestHeadersReader::readHeadersSection(const std::shared_ptr<o
for(v_int32 i = 0; i < res; i ++) {
accumulator <<= 8;
accumulator |= m_buffer[i];
if(accumulator == sectionEnd) {
if(accumulator == SECTION_END) {
result.bufferPosStart = i + 1;
result.bufferPosEnd = (v_int32) res;
return res;

2
test/CMakeLists.txt
View File

@ -37,6 +37,8 @@ add_executable(oatppAllTests
oatpp/parser/json/mapping/DTOMapperTest.hpp
oatpp/parser/json/mapping/DeserializerTest.cpp
oatpp/parser/json/mapping/DeserializerTest.hpp
oatpp/web/mime/multipart/StatefulParserTest.cpp
oatpp/web/mime/multipart/StatefulParserTest.hpp
oatpp/web/server/api/ApiControllerTest.cpp
oatpp/web/server/api/ApiControllerTest.hpp
oatpp/web/FullAsyncTest.cpp

11
test/oatpp/AllTestsMain.cpp
View File

@ -4,6 +4,8 @@
#include "oatpp/web/FullAsyncClientTest.hpp"
#include "oatpp/web/server/api/ApiControllerTest.hpp"
#include "oatpp/web/mime/multipart/StatefulParserTest.hpp"
#include "oatpp/network/virtual_/PipeTest.hpp"
#include "oatpp/network/virtual_/InterfaceTest.hpp"
#include "oatpp/network/UrlTest.hpp"
@ -44,7 +46,7 @@ namespace {
void runTests() {
oatpp::base::Environment::printCompilationConfig();
/*
OATPP_RUN_TEST(oatpp::test::base::RegRuleTest);
OATPP_RUN_TEST(oatpp::test::base::CommandLineArgumentsTest);
@ -71,6 +73,12 @@ void runTests() {
OATPP_RUN_TEST(oatpp::test::network::virtual_::PipeTest);
OATPP_RUN_TEST(oatpp::test::network::virtual_::InterfaceTest);
*/
OATPP_RUN_TEST(oatpp::test::web::mime::multipart::StatefulParserTest);
/*
OATPP_RUN_TEST(oatpp::test::web::server::api::ApiControllerTest);
{
@ -102,6 +110,7 @@ void runTests() {
test_port.run(1);
}
*/
}

74
test/oatpp/web/mime/multipart/StatefulParserTest.cpp Normal file
View File

@ -0,0 +1,74 @@
/***************************************************************************
*
* Project _____ __ ____ _ _
* ( _ ) /__\ (_ _)_| |_ _| |_
* )(_)( /(__)\ )( (_ _)(_ _)
* (_____)(__)(__)(__) |_| |_|
*
*
* Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
***************************************************************************/
#include "StatefulParserTest.hpp"
#include "oatpp/web/mime/multipart/StatefulParser.hpp"
namespace oatpp { namespace test { namespace web { namespace mime { namespace multipart {
namespace {
static const char* TEST_DATA_1 =
"--12345\r\n"
"Content-Disposition: form-data; name=\"part1\"\r\n"
"\r\n"
"part1-value\r\n"
"--12345\r\n"
"Content-Disposition: form-data; name=\"part2\" filename=\"filename.txt\"\r\n"
"\r\n"
"--part2-file-content-line1\r\n"
"--1234part2-file-content-line2\r\n"
"--12345\r\n"
"Content-Disposition: form-data; name=\"part3\" filename=\"filename.jpg\"\r\n"
"\r\n"
"part3-file-binary-data\r\n"
"--12345--\r\n"
;
}
void StatefulParserTest::onRun() {
oatpp::String text = TEST_DATA_1;
{
oatpp::web::mime::multipart::StatefulParser parser("12345");
for (v_int32 i = 0; i < text->getSize(); i++) {
parser.parseNext(&text->getData()[i], 1);
}
}
OATPP_LOGI(TAG, "Test2.................................................");
{
oatpp::web::mime::multipart::StatefulParser parser("12345");
parser.parseNext(text->getData(), text->getSize());
}
}
}}}}}

42
test/oatpp/web/mime/multipart/StatefulParserTest.hpp Normal file
View File

@ -0,0 +1,42 @@
/***************************************************************************
*
* Project _____ __ ____ _ _
* ( _ ) /__\ (_ _)_| |_ _| |_
* )(_)( /(__)\ )( (_ _)(_ _)
* (_____)(__)(__)(__) |_| |_|
*
*
* Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
***************************************************************************/
#ifndef oatpp_test_web_mime_multipart_StatefulParserTest_hpp
#define oatpp_test_web_mime_multipart_StatefulParserTest_hpp
#include "oatpp-test/UnitTest.hpp"
namespace oatpp { namespace test { namespace web { namespace mime { namespace multipart {
class StatefulParserTest : public UnitTest {
public:
StatefulParserTest():UnitTest("TEST[web::mime::multipart::StatefulParserTest]"){}
void onRun() override;
};
}}}}}
#endif /* oatpp_test_web_mime_multipart_StatefulParserTest_hpp */