admin/phoenix-firestorm
1
0
Fork 0
mirror of https://github.com/FirestormViewer/phoenix-firestorm.git synced 2026-08-14 08:53:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Heavily reduce temporary allocations during LLSD parsing operations by utilizing moves and reducing temporary allocations

Browse Source
This commit is contained in:
Rye
2026-01-21 22:07:08 +02:00
committed by Andrey Kleshchev
parent 28fee038e0
commit 4de1ba776c
5 changed files with 226 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
indra/llcommon/llsd.cpp
+128 -7
View File
@@ -39,6 +39,9 @@
#include <limits>
#include <boost/iostreams/device/array.hpp>
#include <boost/iostreams/stream.hpp>
// Defend against a caller forcibly passing a negative number into an unsigned
// size_t index param
inline
@@ -103,6 +106,9 @@ protected:
U32 mUseCount;
public:
static void destruct(Impl*& var);
///< safely decrement or destroy var
static void reset(Impl*& var, Impl* impl);
///< safely set var to refer to the new impl (possibly shared)
@@ -166,7 +172,7 @@ public:
virtual const LLSD& ref(size_t) const { return undef(); }
virtual LLSD::map_const_iterator beginMap() const { return endMap(); }
virtual LLSD::map_const_iterator endMap() const { static const std::map<String, LLSD> empty; return empty.end(); }
virtual LLSD::map_const_iterator endMap() const { static const LLSD::llsd_map_t empty; return empty.end(); }
virtual LLSD::array_const_iterator beginArray() const { return endArray(); }
virtual LLSD::array_const_iterator endArray() const { static const std::vector<LLSD> empty; return empty.end(); }
@@ -346,7 +352,7 @@ namespace
LLSD::Real ImplString::asReal() const
{
F64 v = 0.0;
std::istringstream i_stream(mValue);
boost::iostreams::stream<boost::iostreams::array_source> i_stream(mValue.data(), mValue.size());
i_stream >> v;
// we would probably like to ignore all trailing whitespace as
@@ -431,7 +437,7 @@ namespace
class ImplMap final : public LLSD::Impl
{
private:
typedef std::map<LLSD::String, LLSD, std::less<>> DataMap;
using DataMap = LLSD::llsd_map_t;
DataMap mData;
@@ -457,7 +463,7 @@ namespace
<< it.second.asXMLRPCValue() << "</member>";
}
os << "</struct>";
return os.str();
return std::move(os).str();
}
virtual bool has(std::string_view) const;
@@ -467,8 +473,13 @@ namespace
using LLSD::Impl::ref; // Unhiding ref(size_t)
virtual LLSD get(std::string_view) const;
virtual LLSD getKeys() const;
void insert(std::string&& k, const LLSD& v);
void insert(std::string&& k, LLSD&& v);
void insert(std::string_view k, const LLSD& v);
void insert(std::string_view k, LLSD&& v);
virtual void erase(const LLSD::String&);
LLSD& ref(std::string&&);
virtual const LLSD& ref(std::string&&) const;
LLSD& ref(std::string_view);
virtual const LLSD& ref(std::string_view) const;
@@ -525,18 +536,58 @@ namespace
return keys;
}
void ImplMap::insert(std::string&& k, const LLSD& v)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD;
mData.emplace(std::move(k), v);
}
void ImplMap::insert(std::string&& k, LLSD&& v)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD;
mData.emplace(std::move(k), std::move(v));
}
void ImplMap::insert(std::string_view k, const LLSD& v)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD;
mData.emplace(k, v);
}
void ImplMap::insert(std::string_view k, LLSD&& v)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD;
mData.emplace(k, std::move(v));
}
void ImplMap::erase(const LLSD::String& k)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD;
mData.erase(k);
}
LLSD& ImplMap::ref(std::string&& k)
{
DataMap::iterator i = mData.lower_bound(k);
if (i == mData.end() || mData.key_comp()(k, i->first))
{
return mData.emplace_hint(i, std::make_pair(std::move(k), LLSD()))->second;
}
return i->second;
}
const LLSD& ImplMap::ref(std::string&& k) const
{
DataMap::const_iterator i = mData.lower_bound(k);
if (i == mData.end() || mData.key_comp()(k, i->first))
{
return undef();
}
return i->second;
}
LLSD& ImplMap::ref(std::string_view k)
{
DataMap::iterator i = mData.lower_bound(k);
@@ -598,7 +649,7 @@ namespace
ImplArray(const DataVector& data) : mData(data) { }
public:
ImplArray() { }
ImplArray() = default;
virtual ImplArray& makeArray(Impl*&);
@@ -615,7 +666,7 @@ namespace
os << it.asXMLRPCValue();
}
os << "</data></array>";
return os.str();
return std::move(os).str();
}
using LLSD::Impl::get; // Unhiding get(LLSD::String)
@@ -625,10 +676,13 @@ namespace
virtual LLSD get(size_t) const;
void set(size_t, const LLSD&);
void insert(size_t, const LLSD&);
void insert(size_t, LLSD&&);
LLSD& append(const LLSD&);
LLSD& append(LLSD&&);
virtual void erase(size_t);
LLSD& ref(size_t);
virtual const LLSD& ref(size_t) const;
void reserve(size_t size) { mData.reserve(size); }
LLSD::array_iterator beginArray() { return mData.begin(); }
LLSD::array_iterator endArray() { return mData.end(); }
@@ -690,12 +744,31 @@ namespace
mData.insert(mData.begin() + index, v);
}
void ImplArray::insert(size_t i, LLSD&& v)
{
NEGATIVE_EXIT(i);
DataVector::size_type index = i;
if (index >= mData.size()) // tbd - sanity check limit for index ?
{
mData.resize(index + 1);
}
mData.insert(mData.begin() + index, std::move(v));
}
LLSD& ImplArray::append(const LLSD& v)
{
mData.push_back(v);
return mData.back();
}
LLSD& ImplArray::append(LLSD&& v)
{
mData.push_back(std::move(v));
return mData.back();
}
void ImplArray::erase(size_t i)
{
NEGATIVE_EXIT(i);
@@ -763,6 +836,14 @@ LLSD::Impl::~Impl()
--sOutstandingCount;
}
void LLSD::Impl::destruct(Impl*& var)
{
if (var && var->mUseCount != STATIC_USAGE_COUNT && --var->mUseCount == 0)
{
delete var;
}
}
void LLSD::Impl::reset(Impl*& var, Impl* impl)
{
if (impl && impl->mUseCount != STATIC_USAGE_COUNT)
@@ -961,7 +1042,7 @@ namespace
LLSD::LLSD() : impl(0) { ALLOC_LLSD_OBJECT; }
LLSD::~LLSD() { FREE_LLSD_OBJECT; Impl::reset(impl, 0); }
LLSD::~LLSD() { FREE_LLSD_OBJECT; Impl::destruct(impl); }
LLSD::LLSD(const LLSD& other) : impl(0) { ALLOC_LLSD_OBJECT; assign(other); }
void LLSD::assign(const LLSD& other) { Impl::assign(impl, other.impl); }
@@ -1037,13 +1118,31 @@ LLSD LLSD::emptyMap()
bool LLSD::has(const std::string_view k) const { return safe(impl).has(k); }
LLSD LLSD::get(const std::string_view k) const { return safe(impl).get(k); }
LLSD LLSD::getKeys() const { return safe(impl).getKeys(); }
void LLSD::insert(std::string&& k, const LLSD& v) { makeMap(impl).insert(std::move(k), v); }
void LLSD::insert(std::string&& k, LLSD&& v) { makeMap(impl).insert(std::move(k), std::move(v)); }
void LLSD::insert(std::string_view k, const LLSD& v) { makeMap(impl).insert(k, v); }
void LLSD::insert(std::string_view k, LLSD&& v) { makeMap(impl).insert(k, std::move(v)); }
LLSD& LLSD::with(std::string&& k, const LLSD& v)
{
makeMap(impl).insert(std::move(k), v);
return *this;
}
LLSD& LLSD::with(std::string&& k, LLSD&& v)
{
makeMap(impl).insert(std::move(k), std::move(v));
return *this;
}
LLSD& LLSD::with(std::string_view k, const LLSD& v)
{
makeMap(impl).insert(k, v);
return *this;
}
LLSD& LLSD::with(std::string_view k, LLSD&& v)
{
makeMap(impl).insert(k, std::move(v));
return *this;
}
void LLSD::erase(const String& k) { makeMap(impl).erase(k); }
LLSD& LLSD::operator[](const std::string_view k)
@@ -1051,6 +1150,13 @@ LLSD& LLSD::operator[](const std::string_view k)
LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD;
return makeMap(impl).ref(k);
}
LLSD& LLSD::operator[](std::string&& k)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD;
return makeMap(impl).ref(std::move(k));
}
const LLSD& LLSD::operator[](const std::string_view k) const
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD;
@@ -1064,18 +1170,33 @@ LLSD LLSD::emptyArray()
return v;
}
LLSD LLSD::emptyReservedArray(size_t size)
{
LLSD v;
makeArray(v.impl).reserve(size);
return v;
}
size_t LLSD::size() const { return safe(impl).size(); }
LLSD LLSD::get(Integer i) const { return safe(impl).get(i); }
void LLSD::set(Integer i, const LLSD& v){ makeArray(impl).set(i, v); }
void LLSD::set(Integer i, LLSD&& v) { makeArray(impl).set(i, std::move(v)); }
void LLSD::insert(Integer i, const LLSD& v) { makeArray(impl).insert(i, v); }
void LLSD::insert(Integer i, LLSD&& v) { makeArray(impl).insert(i, std::move(v)); }
LLSD& LLSD::with(Integer i, const LLSD& v)
{
makeArray(impl).insert(i, v);
return *this;
}
LLSD& LLSD::with(Integer i, LLSD&& v)
{
makeArray(impl).insert(i, std::move(v));
return *this;
}
LLSD& LLSD::append(const LLSD& v) { return makeArray(impl).append(v); }
LLSD& LLSD::append(LLSD&& v) { return makeArray(impl).append(std::move(v)); }
void LLSD::erase(Integer i) { makeArray(impl).erase(i); }
LLSD& LLSD::operator[](size_t i)
indra/llcommon/llsd.h
+35 -2
View File
@@ -34,6 +34,7 @@
#include "stdtypes.h"
#include "lldate.h"
#include "llstl.h"
#include "lluri.h"
#include "lluuid.h"
@@ -321,11 +322,34 @@ public:
bool has(const std::string_view) const;
LLSD get(const std::string_view) const;
LLSD getKeys() const; // Return an LLSD array with keys as strings
void insert(const char* k, const LLSD& v)
{
return insert(std::string_view(k), v);
}
void insert(const char* k , LLSD&& v)
{
return insert(std::string_view(k), std::move(v));
}
void insert(std::string&&, const LLSD&);
void insert(std::string&&, LLSD&&);
void insert(std::string_view, const LLSD&);
void insert(std::string_view, LLSD&&);
void erase(const String&);
LLSD& with(const char* k, const LLSD& v)
{
return with(std::string_view(k), v);
}
LLSD& with(const char* k, LLSD&& v)
{
return with(std::string_view(k), std::move(v));
}
LLSD& with(std::string&&, const LLSD&);
LLSD& with(std::string&&, LLSD&&);
LLSD& with(std::string_view, const LLSD&);
LLSD& with(std::string_view, LLSD&&);
LLSD& operator[](const std::string_view);
LLSD& operator[](std::string&&);
LLSD& operator[](const char* c)
{
return c ? (*this)[std::string_view(c)] : *this;
@@ -339,14 +363,22 @@ public:
/** @name Array Values */
//@{
// Allocate an empty array
static LLSD emptyArray();
// Allocate an array with internal storage reserved but not initialized like a std::vector
static LLSD emptyReservedArray(size_t size);
LLSD get(Integer) const;
void set(Integer, const LLSD&);
void set(Integer, LLSD&&);
void insert(Integer, const LLSD&);
void insert(Integer, LLSD&&);
LLSD& append(const LLSD&);
LLSD& append(LLSD&&);
void erase(Integer);
LLSD& with(Integer, const LLSD&);
LLSD& with(Integer, LLSD&&);
// accept size_t so we can index relative to size()
const LLSD& operator[](size_t) const;
@@ -366,8 +398,9 @@ public:
//@{
size_t size() const;
typedef std::map<String, LLSD>::iterator map_iterator;
typedef std::map<String, LLSD>::const_iterator map_const_iterator;
using llsd_map_t = std::map<String, LLSD, std::less<>>;
typedef llsd_map_t::iterator map_iterator;
typedef llsd_map_t::const_iterator map_const_iterator;
map_iterator beginMap();
map_iterator endMap();
indra/llcommon/llsdjson.cpp
+2 -2
View File
@@ -66,7 +66,7 @@ LLSD LlsdFromJson(const boost::json::value& val)
const boost::json::array& array = val.as_array();
size_t size = array.size();
// allocate elements 0 .. (size() - 1) to avoid incremental allocation
if (! array.empty())
if (!array.empty())
{
result[size - 1] = LLSD();
}
@@ -80,7 +80,7 @@ LLSD LlsdFromJson(const boost::json::value& val)
result = LLSD::emptyMap();
for (const auto& element : val.as_object())
{
result[element.key()] = LlsdFromJson(element.value());
result[std::string_view(element.key())] = LlsdFromJson(element.value());
}
break;
}
indra/llcommon/llsdserialize.cpp
+23 -29
View File
@@ -775,7 +775,8 @@ S32 LLSDNotationParser::parseMap(std::istream& istr, LLSD& map, S32 max_depth) c
// There must be a value for every key, thus
// child_count must be greater than 0.
parse_count += count;
map.insert(name, child);
map.insert(std::move(name), std::move(child)); // Move as name will be filled on next iteration
name.clear();
}
else
{
@@ -822,7 +823,7 @@ S32 LLSDNotationParser::parseArray(std::istream& istr, LLSD& array, S32 max_dept
else
{
parse_count += count;
array.append(child);
array.append(std::move(child));
}
c = get(istr);
}
@@ -841,7 +842,7 @@ bool LLSDNotationParser::parseString(std::istream& istr, LLSD& data) const
auto count = deserialize_string(istr, value, mMaxBytesLeft);
if(PARSE_FAILURE == count) return false;
account(count);
data = value;
data = std::move(value);
return true;
}
@@ -872,10 +873,10 @@ bool LLSDNotationParser::parseBinary(std::istream& istr, LLSD& data) const
if(len)
{
value.resize(len);
account(fullread(istr, (char *)&value[0], len));
account(fullread(istr, (char*)value.data(), len));
}
c = get(istr); // strip off the trailing double-quote
data = value;
data = std::move(value);
}
else if(0 == strncmp("b64", buf, 3))
{
@@ -885,7 +886,7 @@ bool LLSDNotationParser::parseBinary(std::istream& istr, LLSD& data) const
std::stringstream coded_stream;
get(istr, *(coded_stream.rdbuf()), '\"');
c = get(istr);
std::string encoded(coded_stream.str());
std::string encoded(std::move(coded_stream).str());
S32 len = apr_base64_decode_len(encoded.c_str());
std::vector<U8> value;
if(len)
@@ -894,7 +895,7 @@ bool LLSDNotationParser::parseBinary(std::istream& istr, LLSD& data) const
len = apr_base64_decode_binary(&value[0], encoded.c_str());
value.resize(len);
}
data = value;
data = std::move(value);
}
else if(0 == strncmp("b16", buf, 3))
{
@@ -925,7 +926,7 @@ bool LLSDNotationParser::parseBinary(std::istream& istr, LLSD& data) const
// copy the data out of the byte buffer
value.insert(value.end(), byte_buffer, write);
}
data = value;
data = std::move(value);
}
else
{
@@ -1077,7 +1078,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) con
}
else
{
data = value;
data = std::move(value);
account(cnt);
}
if(istr.fail())
@@ -1094,7 +1095,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) con
std::string value;
if(parseString(istr, value))
{
data = value;
data = std::move(value);
}
else
{
@@ -1159,7 +1160,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) con
value.resize(size);
account(fullread(istr, (char*)&value[0], size));
}
data = value;
data = std::move(value);
}
if(istr.fail())
{
@@ -1218,7 +1219,7 @@ S32 LLSDBinaryParser::parseMap(std::istream& istr, LLSD& map, S32 max_depth) con
// There must be a value for every key, thus child_count
// must be greater than 0.
parse_count += child_count;
map.insert(name, child);
map.insert(std::move(name), std::move(child));
}
else
{
@@ -1238,13 +1239,12 @@ S32 LLSDBinaryParser::parseMap(std::istream& istr, LLSD& map, S32 max_depth) con
S32 LLSDBinaryParser::parseArray(std::istream& istr, LLSD& array, S32 max_depth) const
{
array = LLSD::emptyArray();
U32 value_nbo = 0;
read(istr, (char*)&value_nbo, sizeof(U32)); /*Flawfinder: ignore*/
S32 size = (S32)ntohl(value_nbo);
// *FIX: This would be a good place to reserve some space in the
// array...
// Preallocate array to avoid incremental allocation
array = LLSD::emptyReservedArray(size);
S32 parse_count = 0;
S32 count = 0;
@@ -1260,7 +1260,7 @@ S32 LLSDBinaryParser::parseArray(std::istream& istr, LLSD& array, S32 max_depth)
if(child_count)
{
parse_count += child_count;
array.append(child);
array.append(std::move(child));
}
++count;
c = istr.peek();
@@ -1279,18 +1279,15 @@ bool LLSDBinaryParser::parseString(
std::istream& istr,
std::string& value) const
{
// *FIX: This is memory inefficient.
U32 value_nbo = 0;
read(istr, (char*)&value_nbo, sizeof(U32)); /*Flawfinder: ignore*/
S32 size = (S32)ntohl(value_nbo);
if(mCheckLimits && (size > mMaxBytesLeft)) return false;
if(size < 0) return false;
std::vector<char> buf;
if(size)
{
buf.resize(size);
account(fullread(istr, &buf[0], size));
value.assign(buf.begin(), buf.end());
value.resize(size);
account(fullread(istr, value.data(), size));
}
return true;
}
@@ -1785,7 +1782,7 @@ llssize deserialize_string_delim(
}
}
value = write_buffer.str();
value = std::move(write_buffer).str();
return count;
}
@@ -1806,15 +1803,12 @@ llssize deserialize_string_raw(
{
// We probably have a valid raw string. determine
// the size, and read it.
// *FIX: This is memory inefficient.
auto len = strtol(buf + 1, NULL, 0);
auto len = strtol(buf + 1, nullptr, 0);
if((max_bytes>0)&&(len>max_bytes)) return LLSDParser::PARSE_FAILURE;
std::vector<char> buf;
if(len)
{
buf.resize(len);
count += fullread(istr, (char *)&buf[0], len);
value.assign(buf.begin(), buf.end());
value.resize(len);
count += fullread(istr, value.data(), len);
}
c = istr.get();
++count;
@@ -2170,7 +2164,7 @@ std::string zip_llsd(LLSD& data)
return std::string();
}
std::string source = llsd_strm.str();
std::string source = std::move(llsd_strm).str();
U8 out[CHUNK];
indra/llcommon/llsdserialize_xml.cpp
+38 -11
View File
@@ -31,6 +31,8 @@
#include <deque>
#include "apr_base64.h"
#include <boost/iostreams/device/array.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/regex.hpp>
extern "C"
@@ -645,7 +647,7 @@ void LLSDXMLParser::Impl::startElementHandler(const XML_Char* name, const XML_Ch
if (mCurrentKey.empty()) { return startSkipping(); }
LLSD& map = *mStack.back();
LLSD& newElement = map[mCurrentKey];
LLSD& newElement = map[std::move(mCurrentKey)];
mStack.push_back(&newElement);
mCurrentKey.clear();
@@ -709,7 +711,8 @@ void LLSDXMLParser::Impl::endElementHandler(const XML_Char* name)
return;
case ELEMENT_KEY:
mCurrentKey = mCurrentContent;
mCurrentKey = std::move(mCurrentContent); // This is safe to move as we are in the end element handler
mCurrentContent.clear(); // Clear to reset to valid state
return;
default:
@@ -742,14 +745,39 @@ void LLSDXMLParser::Impl::endElementHandler(const XML_Char* name)
}
else
{
value = LLSD(mCurrentContent).asInteger();
// This implementation is copied from llsd.cpp
F64 v = 0.0;
boost::iostreams::stream<boost::iostreams::array_source> i_stream(mCurrentContent.data(), mCurrentContent.size());
i_stream >> v;
// we would probably like to ignore all trailing whitespace as
// well, but for now, simply eat the next character, and make
// sure we reached the end of the string.
// *NOTE: gcc 2.95 does not generate an eof() event on the
// stream operation above, so we manually get here to force it
// across platforms.
int c = i_stream.get();
value = (int)((EOF == c) ? v : 0.0);
}
}
break;
case ELEMENT_REAL:
{
value = LLSD(mCurrentContent).asReal();
// This implementation is copied from llsd.cpp
F64 v = 0.0;
boost::iostreams::stream<boost::iostreams::array_source> i_stream(mCurrentContent.data(), mCurrentContent.size());
i_stream >> v;
// we would probably like to ignore all trailing whitespace as
// well, but for now, simply eat the next character, and make
// sure we reached the end of the string.
// *NOTE: gcc 2.95 does not generate an eof() event on the
// stream operation above, so we manually get here to force it
// across platforms.
int c = i_stream.get();
value = ((EOF == c) ? v : 0.0);
// removed since this breaks when locale has decimal separator that isn't '.'
// investigated changing local to something compatible each time but deemed higher
// risk that just using LLSD.asReal() each time.
@@ -766,19 +794,19 @@ void LLSDXMLParser::Impl::endElementHandler(const XML_Char* name)
break;
case ELEMENT_STRING:
value = mCurrentContent;
value = std::move(mCurrentContent); // This is safe to move as we are in the end element handler and this is cleared below
break;
case ELEMENT_UUID:
value = LLSD(mCurrentContent).asUUID();
value = LLUUID(mCurrentContent);
break;
case ELEMENT_DATE:
value = LLSD(mCurrentContent).asDate();
value = LLDate(mCurrentContent);
break;
case ELEMENT_URI:
value = LLSD(mCurrentContent).asURI();
value = LLURI(mCurrentContent);
break;
case ELEMENT_BINARY:
@@ -787,15 +815,14 @@ void LLSDXMLParser::Impl::endElementHandler(const XML_Char* name)
// created by python and other non-linden systems - DEV-39358
// Fortunately we have very little binary passing now,
// so performance impact shold be negligible. + poppy 2009-09-04
boost::regex r;
r.assign("\\s");
static const boost::regex r("\\s");
std::string stripped = boost::regex_replace(mCurrentContent, r, "");
S32 len = apr_base64_decode_len(stripped.c_str());
std::vector<U8> data;
data.resize(len);
len = apr_base64_decode_binary(&data[0], stripped.c_str());
data.resize(len);
value = data;
value = std::move(data);
break;
}