utfiterator.h
Go to the documentation of this file.
// © 2024 and later: Unicode, Inc. and others.
// License & terms of use: https://www.unicode.org/copyright.html
// utfiterator.h
// created: 2024aug12 Markus W. Scherer
#ifndef __UTFITERATOR_H__
#define __UTFITERATOR_H__
#include "unicode/utypes.h"
#if U_SHOW_CPLUSPLUS_API || U_SHOW_CPLUSPLUS_HEADER_API || !defined(UTYPES_H)
#include <iterator>
#if defined(__cpp_lib_ranges)
#include <ranges>
#endif
#include <string>
#include <string_view>
#include <type_traits>
#include "unicode/utf16.h"
#include "unicode/utf8.h"
#include "unicode/uversion.h"
#ifndef U_HIDE_DRAFT_API
typedef enum UTFIllFormedBehavior {
UTF_BEHAVIOR_NEGATIVE,
UTF_BEHAVIOR_FFFD,
UTF_BEHAVIOR_SURROGATE
} UTFIllFormedBehavior;
namespace U_HEADER_ONLY_NAMESPACE {
namespace prv {
#if U_CPLUSPLUS_VERSION >= 20
template<typename Iter>
using iter_value_t = typename std::iter_value_t<Iter>;
template<typename Iter>
using iter_difference_t = std::iter_difference_t<Iter>;
template<typename Iter>
constexpr bool forward_iterator = std::forward_iterator<Iter>;
template<typename Iter>
constexpr bool bidirectional_iterator = std::bidirectional_iterator<Iter>;
template<typename Range>
constexpr bool range = std::ranges::range<Range>;
#else
template<typename Iter>
using iter_value_t = typename std::iterator_traits<Iter>::value_type;
template<typename Iter>
using iter_difference_t = typename std::iterator_traits<Iter>::difference_type;
template<typename Iter>
constexpr bool forward_iterator =
std::is_base_of_v<
std::forward_iterator_tag,
typename std::iterator_traits<Iter>::iterator_category>;
template<typename Iter>
constexpr bool bidirectional_iterator =
std::is_base_of_v<
std::bidirectional_iterator_tag,
typename std::iterator_traits<Iter>::iterator_category>;
template<typename Range, typename = void>
struct range_type : std::false_type {};
template<typename Range>
struct range_type<
Range,
std::void_t<decltype(std::declval<Range>().begin()),
decltype(std::declval<Range>().end())>> : std::true_type {};
template<typename Range>
constexpr bool range = range_type<Range>::value;
#endif
template <typename T> struct is_basic_string_view : std::false_type {};
template <typename... Args>
struct is_basic_string_view<std::basic_string_view<Args...>> : std::true_type {};
template <typename T> constexpr bool is_basic_string_view_v = is_basic_string_view<T>::value;
template<typename CP32, bool skipSurrogates>
class CodePointsIterator {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
public:
using value_type = CP32;
using reference = value_type;
using pointer = CP32 *;
using difference_type = int32_t;
using iterator_category = std::forward_iterator_tag;
inline CodePointsIterator(CP32 c) : c_(c) {}
inline bool operator==(const CodePointsIterator &other) const { return c_ == other.c_; }
inline bool operator!=(const CodePointsIterator &other) const { return !operator==(other); }
inline CP32 operator*() const { return c_; }
inline CodePointsIterator &operator++() { // pre-increment
++c_;
if (skipSurrogates && c_ == 0xd800) {
c_ = 0xe000;
}
return *this;
}
inline CodePointsIterator operator++(int) { // post-increment
CodePointsIterator result(*this);
++(*this);
return result;
}
private:
CP32 c_;
};
} // namespace prv
template<typename CP32>
class AllCodePoints {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
public:
AllCodePoints() {}
auto begin() const { return prv::CodePointsIterator<CP32, false>(0); }
auto end() const { return prv::CodePointsIterator<CP32, false>(0x110000); }
};
template<typename CP32>
class AllScalarValues {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
public:
AllScalarValues() {}
auto begin() const { return prv::CodePointsIterator<CP32, true>(0); }
auto end() const { return prv::CodePointsIterator<CP32, true>(0x110000); }
};
template<typename CP32, typename UnitIter, typename = void>
class UnsafeCodeUnits {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
using Unit = typename prv::iter_value_t<UnitIter>;
public:
UnsafeCodeUnits(CP32 codePoint, uint8_t length, UnitIter start, UnitIter limit) :
c_(codePoint), len_(length), start_(start), limit_(limit) {}
UnsafeCodeUnits(const UnsafeCodeUnits &other) = default;
UnsafeCodeUnits &operator=(const UnsafeCodeUnits &other) = default;
CP32 codePoint() const { return c_; }
UnitIter begin() const { return start_; }
UnitIter end() const { return limit_; }
uint8_t length() const { return len_; }
#if U_CPLUSPLUS_VERSION >= 20
template<std::contiguous_iterator Iter = UnitIter>
std::basic_string_view<Unit> stringView() const {
return std::basic_string_view<Unit>(begin(), end());
}
#else
template<typename Iter = UnitIter, typename Unit = typename std::iterator_traits<Iter>::value_type>
std::enable_if_t<std::is_pointer_v<Iter> ||
std::is_same_v<Iter, typename std::basic_string<Unit>::iterator> ||
std::is_same_v<Iter, typename std::basic_string<Unit>::const_iterator> ||
std::is_same_v<Iter, typename std::basic_string_view<Unit>::iterator> ||
std::is_same_v<Iter, typename std::basic_string_view<Unit>::const_iterator>,
std::basic_string_view<Unit>>
stringView() const {
return std::basic_string_view<Unit>(&*start_, len_);
}
#endif
private:
// Order of fields with padding and access frequency in mind.
CP32 c_;
uint8_t len_;
UnitIter start_;
UnitIter limit_;
};
#ifndef U_IN_DOXYGEN
// Partial template specialization for single-pass input iterator.
// No UnitIter field, no getter for it, no stringView().
template<typename CP32, typename UnitIter>
class UnsafeCodeUnits<
CP32,
UnitIter,
std::enable_if_t<!prv::forward_iterator<UnitIter>>> {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
public:
UnsafeCodeUnits(CP32 codePoint, uint8_t length) : c_(codePoint), len_(length) {}
UnsafeCodeUnits(const UnsafeCodeUnits &other) = default;
UnsafeCodeUnits &operator=(const UnsafeCodeUnits &other) = default;
CP32 codePoint() const { return c_; }
uint8_t length() const { return len_; }
private:
// Order of fields with padding and access frequency in mind.
CP32 c_;
uint8_t len_;
};
#endif // U_IN_DOXYGEN
template<typename CP32, typename UnitIter, typename = void>
class CodeUnits : public UnsafeCodeUnits<CP32, UnitIter> {
public:
CodeUnits(CP32 codePoint, uint8_t length, bool wellFormed, UnitIter start, UnitIter limit) :
UnsafeCodeUnits<CP32, UnitIter>(codePoint, length, start, limit), ok_(wellFormed) {}
CodeUnits(const CodeUnits &other) = default;
CodeUnits &operator=(const CodeUnits &other) = default;
bool wellFormed() const { return ok_; }
private:
bool ok_;
};
#ifndef U_IN_DOXYGEN
// Partial template specialization for single-pass input iterator.
// No UnitIter field, no getter for it, no stringView().
template<typename CP32, typename UnitIter>
class CodeUnits<
CP32,
UnitIter,
std::enable_if_t<!prv::forward_iterator<UnitIter>>> :
public UnsafeCodeUnits<CP32, UnitIter> {
public:
CodeUnits(CP32 codePoint, uint8_t length, bool wellFormed) :
UnsafeCodeUnits<CP32, UnitIter>(codePoint, length), ok_(wellFormed) {}
CodeUnits(const CodeUnits &other) = default;
CodeUnits &operator=(const CodeUnits &other) = default;
bool wellFormed() const { return ok_; }
private:
bool ok_;
};
#endif // U_IN_DOXYGEN
// Validating implementations ---------------------------------------------- ***
#ifndef U_IN_DOXYGEN
template<typename CP32, UTFIllFormedBehavior behavior,
typename UnitIter, typename LimitIter = UnitIter, typename = void>
class UTFImpl;
// Note: readAndInc() functions take both a p0 and a p iterator.
// They must have the same value.
// For a multi-pass UnitIter, the caller must copy its p into a local variable p0,
// and readAndInc() copies p0 and the incremented p into the CodeUnits.
// For a single-pass UnitIter, which may not be default-constructible nor coypable,
// the caller can pass p into both references, and readAndInc() does not use p0
// and constructs CodeUnits without them.
// Moving the p0 variable into the call site avoids having to declare it inside readAndInc()
// which may not be possible for a single-pass iterator.
// UTF-8
template<typename CP32, UTFIllFormedBehavior behavior, typename UnitIter, typename LimitIter>
class UTFImpl<
CP32, behavior,
UnitIter, LimitIter,
std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 1>> {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
static_assert(behavior != UTF_BEHAVIOR_SURROGATE,
"For 8-bit strings, the SURROGATE option does not have an equivalent.");
public:
// Handle ill-formed UTF-8
U_FORCE_INLINE static CP32 sub() {
switch (behavior) {
case UTF_BEHAVIOR_NEGATIVE: return U_SENTINEL;
case UTF_BEHAVIOR_FFFD: return 0xfffd;
}
}
U_FORCE_INLINE static void inc(UnitIter &p, const LimitIter &limit) {
// Very similar to U8_FWD_1().
uint8_t b = *p;
++p;
if (U8_IS_LEAD(b) && p != limit) {
uint8_t t1 = *p;
if ((0xe0 <= b && b < 0xf0)) {
if (U8_IS_VALID_LEAD3_AND_T1(b, t1) &&
++p != limit && U8_IS_TRAIL(*p)) {
++p;
}
} else if (b < 0xe0) {
if (U8_IS_TRAIL(t1)) {
++p;
}
} else /* b >= 0xf0 */ {
if (U8_IS_VALID_LEAD4_AND_T1(b, t1) &&
++p != limit && U8_IS_TRAIL(*p) &&
++p != limit && U8_IS_TRAIL(*p)) {
++p;
}
}
}
}
U_FORCE_INLINE static void dec(UnitIter start, UnitIter &p) {
// Very similar to U8_BACK_1().
uint8_t c = *--p;
if (U8_IS_TRAIL(c) && p != start) {
UnitIter p1 = p;
uint8_t b1 = *--p1;
if (U8_IS_LEAD(b1)) {
if (b1 < 0xe0 ||
(b1 < 0xf0 ?
U8_IS_VALID_LEAD3_AND_T1(b1, c) :
U8_IS_VALID_LEAD4_AND_T1(b1, c))) {
p = p1;
return;
}
} else if (U8_IS_TRAIL(b1) && p1 != start) {
uint8_t b2 = *--p1;
if (0xe0 <= b2 && b2 <= 0xf4) {
if (b2 < 0xf0 ?
U8_IS_VALID_LEAD3_AND_T1(b2, b1) :
U8_IS_VALID_LEAD4_AND_T1(b2, b1)) {
p = p1;
return;
}
} else if (U8_IS_TRAIL(b2) && p1 != start) {
uint8_t b3 = *--p1;
if (0xf0 <= b3 && b3 <= 0xf4 && U8_IS_VALID_LEAD4_AND_T1(b3, b2)) {
p = p1;
return;
}
}
}
}
}
U_FORCE_INLINE static CodeUnits<CP32, UnitIter> readAndInc(
UnitIter &p0, UnitIter &p, const LimitIter &limit) {
constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
// Very similar to U8_NEXT_OR_FFFD().
CP32 c = uint8_t(*p);
++p;
if (U8_IS_SINGLE(c)) {
if constexpr (isMultiPass) {
return {c, 1, true, p0, p};
} else {
return {c, 1, true};
}
}
uint8_t length = 1;
uint8_t t = 0;
if (p != limit &&
// fetch/validate/assemble all but last trail byte
(c >= 0xe0 ?
(c < 0xf0 ? // U+0800..U+FFFF except surrogates
U8_LEAD3_T1_BITS[c &= 0xf] & (1 << ((t = *p) >> 5)) &&
(t &= 0x3f, 1)
: // U+10000..U+10FFFF
(c -= 0xf0) <= 4 &&
U8_LEAD4_T1_BITS[(t = *p) >> 4] & (1 << c) &&
(c = (c << 6) | (t & 0x3f), ++length, ++p != limit) &&
(t = *p - 0x80) <= 0x3f) &&
// valid second-to-last trail byte
(c = (c << 6) | t, ++length, ++p != limit)
: // U+0080..U+07FF
c >= 0xc2 && (c &= 0x1f, 1)) &&
// last trail byte
(t = *p - 0x80) <= 0x3f) {
c = (c << 6) | t;
++length;
++p;
if constexpr (isMultiPass) {
return {c, length, true, p0, p};
} else {
return {c, length, true};
}
}
if constexpr (isMultiPass) {
return {sub(), length, false, p0, p};
} else {
return {sub(), length, false};
}
}
U_FORCE_INLINE static CodeUnits<CP32, UnitIter> decAndRead(UnitIter start, UnitIter &p) {
// Very similar to U8_PREV_OR_FFFD().
UnitIter p0 = p;
CP32 c = uint8_t(*--p);
if (U8_IS_SINGLE(c)) {
return {c, 1, true, p, p0};
}
if (U8_IS_TRAIL(c) && p != start) {
UnitIter p1 = p;
uint8_t b1 = *--p1;
if (U8_IS_LEAD(b1)) {
if (b1 < 0xe0) {
p = p1;
c = ((b1 - 0xc0) << 6) | (c & 0x3f);
return {c, 2, true, p, p0};
} else if (b1 < 0xf0 ?
U8_IS_VALID_LEAD3_AND_T1(b1, c) :
U8_IS_VALID_LEAD4_AND_T1(b1, c)) {
// Truncated 3- or 4-byte sequence.
p = p1;
return {sub(), 2, false, p, p0};
}
} else if (U8_IS_TRAIL(b1) && p1 != start) {
// Extract the value bits from the last trail byte.
c &= 0x3f;
uint8_t b2 = *--p1;
if (0xe0 <= b2 && b2 <= 0xf4) {
if (b2 < 0xf0) {
b2 &= 0xf;
if (U8_IS_VALID_LEAD3_AND_T1(b2, b1)) {
p = p1;
c = (b2 << 12) | ((b1 & 0x3f) << 6) | c;
return {c, 3, true, p, p0};
}
} else if (U8_IS_VALID_LEAD4_AND_T1(b2, b1)) {
// Truncated 4-byte sequence.
p = p1;
return {sub(), 3, false, p, p0};
}
} else if (U8_IS_TRAIL(b2) && p1 != start) {
uint8_t b3 = *--p1;
if (0xf0 <= b3 && b3 <= 0xf4) {
b3 &= 7;
if (U8_IS_VALID_LEAD4_AND_T1(b3, b2)) {
p = p1;
c = (b3 << 18) | ((b2 & 0x3f) << 12) | ((b1 & 0x3f) << 6) | c;
return {c, 4, true, p, p0};
}
}
}
}
}
return {sub(), 1, false, p, p0};
}
};
// UTF-16
template<typename CP32, UTFIllFormedBehavior behavior, typename UnitIter, typename LimitIter>
class UTFImpl<
CP32, behavior,
UnitIter, LimitIter,
std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 2>> {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
public:
// Handle ill-formed UTF-16: One unpaired surrogate.
U_FORCE_INLINE static CP32 sub(CP32 surrogate) {
switch (behavior) {
case UTF_BEHAVIOR_NEGATIVE: return U_SENTINEL;
case UTF_BEHAVIOR_FFFD: return 0xfffd;
case UTF_BEHAVIOR_SURROGATE: return surrogate;
}
}
U_FORCE_INLINE static void inc(UnitIter &p, const LimitIter &limit) {
// Very similar to U16_FWD_1().
auto c = *p;
++p;
if (U16_IS_LEAD(c) && p != limit && U16_IS_TRAIL(*p)) {
++p;
}
}
U_FORCE_INLINE static void dec(UnitIter start, UnitIter &p) {
// Very similar to U16_BACK_1().
UnitIter p1;
if (U16_IS_TRAIL(*--p) && p != start && (p1 = p, U16_IS_LEAD(*--p1))) {
p = p1;
}
}
U_FORCE_INLINE static CodeUnits<CP32, UnitIter> readAndInc(
UnitIter &p0, UnitIter &p, const LimitIter &limit) {
constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
// Very similar to U16_NEXT_OR_FFFD().
CP32 c = static_cast<CP32>(*p);
++p;
if (!U16_IS_SURROGATE(c)) {
if constexpr (isMultiPass) {
return {c, 1, true, p0, p};
} else {
return {c, 1, true};
}
} else {
uint16_t c2;
if (U16_IS_SURROGATE_LEAD(c) && p != limit && U16_IS_TRAIL(c2 = *p)) {
++p;
c = U16_GET_SUPPLEMENTARY(c, c2);
if constexpr (isMultiPass) {
return {c, 2, true, p0, p};
} else {
return {c, 2, true};
}
} else {
if constexpr (isMultiPass) {
return {sub(c), 1, false, p0, p};
} else {
return {sub(c), 1, false};
}
}
}
}
U_FORCE_INLINE static CodeUnits<CP32, UnitIter> decAndRead(UnitIter start, UnitIter &p) {
// Very similar to U16_PREV_OR_FFFD().
UnitIter p0 = p;
CP32 c = static_cast<CP32>(*--p);
if (!U16_IS_SURROGATE(c)) {
return {c, 1, true, p, p0};
} else {
UnitIter p1;
uint16_t c2;
if (U16_IS_SURROGATE_TRAIL(c) && p != start && (p1 = p, U16_IS_LEAD(c2 = *--p1))) {
p = p1;
c = U16_GET_SUPPLEMENTARY(c2, c);
return {c, 2, true, p, p0};
} else {
return {sub(c), 1, false, p, p0};
}
}
}
};
// UTF-32: trivial, but still validating
template<typename CP32, UTFIllFormedBehavior behavior, typename UnitIter, typename LimitIter>
class UTFImpl<
CP32, behavior,
UnitIter, LimitIter,
std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 4>> {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
public:
// Handle ill-formed UTF-32
U_FORCE_INLINE static CP32 sub(bool forSurrogate, CP32 surrogate) {
switch (behavior) {
case UTF_BEHAVIOR_NEGATIVE: return U_SENTINEL;
case UTF_BEHAVIOR_FFFD: return 0xfffd;
case UTF_BEHAVIOR_SURROGATE: return forSurrogate ? surrogate : 0xfffd;
}
}
U_FORCE_INLINE static void inc(UnitIter &p, const LimitIter &/*limit*/) {
++p;
}
U_FORCE_INLINE static void dec(UnitIter /*start*/, UnitIter &p) {
--p;
}
U_FORCE_INLINE static CodeUnits<CP32, UnitIter> readAndInc(
UnitIter &p0, UnitIter &p, const LimitIter &/*limit*/) {
constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
uint32_t uc = *p;
CP32 c = uc;
++p;
if (uc < 0xd800 || (0xe000 <= uc && uc <= 0x10ffff)) {
if constexpr (isMultiPass) {
return {c, 1, true, p0, p};
} else {
return {c, 1, true};
}
} else {
if constexpr (isMultiPass) {
return {sub(uc < 0xe000, c), 1, false, p0, p};
} else {
return {sub(uc < 0xe000, c), 1, false};
}
}
}
U_FORCE_INLINE static CodeUnits<CP32, UnitIter> decAndRead(UnitIter /*start*/, UnitIter &p) {
UnitIter p0 = p;
uint32_t uc = *--p;
CP32 c = uc;
if (uc < 0xd800 || (0xe000 <= uc && uc <= 0x10ffff)) {
return {c, 1, true, p, p0};
} else {
return {sub(uc < 0xe000, c), 1, false, p, p0};
}
}
};
// Non-validating implementations ------------------------------------------ ***
template<typename CP32, typename UnitIter, typename = void>
class UnsafeUTFImpl;
// UTF-8
template<typename CP32, typename UnitIter>
class UnsafeUTFImpl<
CP32,
UnitIter,
std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 1>> {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
public:
U_FORCE_INLINE static void inc(UnitIter &p) {
// Very similar to U8_FWD_1_UNSAFE().
uint8_t b = *p;
std::advance(p, 1 + U8_COUNT_TRAIL_BYTES_UNSAFE(b));
}
U_FORCE_INLINE static void dec(UnitIter &p) {
// Very similar to U8_BACK_1_UNSAFE().
while (U8_IS_TRAIL(*--p)) {}
}
U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> readAndInc(UnitIter &p0, UnitIter &p) {
constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
// Very similar to U8_NEXT_UNSAFE().
CP32 c = uint8_t(*p);
++p;
if (U8_IS_SINGLE(c)) {
if constexpr (isMultiPass) {
return {c, 1, p0, p};
} else {
return {c, 1};
}
} else if (c < 0xe0) {
c = ((c & 0x1f) << 6) | (*p & 0x3f);
++p;
if constexpr (isMultiPass) {
return {c, 2, p0, p};
} else {
return {c, 2};
}
} else if (c < 0xf0) {
// No need for (c&0xf) because the upper bits are truncated
// after <<12 in the cast to uint16_t.
c = uint16_t(c << 12) | ((*p & 0x3f) << 6);
++p;
c |= *p & 0x3f;
++p;
if constexpr (isMultiPass) {
return {c, 3, p0, p};
} else {
return {c, 3};
}
} else {
c = ((c & 7) << 18) | ((*p & 0x3f) << 12);
++p;
c |= (*p & 0x3f) << 6;
++p;
c |= *p & 0x3f;
++p;
if constexpr (isMultiPass) {
return {c, 4, p0, p};
} else {
return {c, 4};
}
}
}
U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> decAndRead(UnitIter &p) {
// Very similar to U8_PREV_UNSAFE().
UnitIter p0 = p;
CP32 c = uint8_t(*--p);
if (U8_IS_SINGLE(c)) {
return {c, 1, p, p0};
}
// U8_IS_TRAIL(c) if well-formed
c &= 0x3f;
uint8_t count = 1;
for (uint8_t shift = 6;;) {
uint8_t b = *--p;
if (b >= 0xc0) {
U8_MASK_LEAD_BYTE(b, count);
c |= uint32_t{b} << shift;
break;
} else {
c |= (uint32_t{b} & 0x3f) << shift;
++count;
shift += 6;
}
}
++count;
return {c, count, p, p0};
}
};
// UTF-16
template<typename CP32, typename UnitIter>
class UnsafeUTFImpl<
CP32,
UnitIter,
std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 2>> {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
public:
U_FORCE_INLINE static void inc(UnitIter &p) {
// Very similar to U16_FWD_1_UNSAFE().
auto c = *p;
++p;
if (U16_IS_LEAD(c)) {
++p;
}
}
U_FORCE_INLINE static void dec(UnitIter &p) {
// Very similar to U16_BACK_1_UNSAFE().
if (U16_IS_TRAIL(*--p)) {
--p;
}
}
U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> readAndInc(UnitIter &p0, UnitIter &p) {
constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
// Very similar to U16_NEXT_UNSAFE().
CP32 c = static_cast<CP32>(*p);
++p;
if (!U16_IS_LEAD(c)) {
if constexpr (isMultiPass) {
return {c, 1, p0, p};
} else {
return {c, 1};
}
} else {
uint16_t c2 = *p;
++p;
c = U16_GET_SUPPLEMENTARY(c, c2);
if constexpr (isMultiPass) {
return {c, 2, p0, p};
} else {
return {c, 2};
}
}
}
U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> decAndRead(UnitIter &p) {
// Very similar to U16_PREV_UNSAFE().
UnitIter p0 = p;
CP32 c = static_cast<CP32>(*--p);
if (!U16_IS_TRAIL(c)) {
return {c, 1, p, p0};
} else {
uint16_t c2 = *--p;
c = U16_GET_SUPPLEMENTARY(c2, c);
return {c, 2, p, p0};
}
}
};
// UTF-32: trivial
template<typename CP32, typename UnitIter>
class UnsafeUTFImpl<
CP32,
UnitIter,
std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 4>> {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
public:
U_FORCE_INLINE static void inc(UnitIter &p) {
++p;
}
U_FORCE_INLINE static void dec(UnitIter &p) {
--p;
}
U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> readAndInc(UnitIter &p0, UnitIter &p) {
constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
CP32 c = *p;
++p;
if constexpr (isMultiPass) {
return {c, 1, p0, p};
} else {
return {c, 1};
}
}
U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> decAndRead(UnitIter &p) {
UnitIter p0 = p;
CP32 c = *--p;
return {c, 1, p, p0};
}
};
#endif
// Validating iterators ---------------------------------------------------- ***
template<typename CP32, UTFIllFormedBehavior behavior,
typename UnitIter, typename LimitIter = UnitIter, typename = void>
class UTFIterator {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
using Impl = UTFImpl<CP32, behavior, UnitIter, LimitIter>;
// Proxy type for operator->() (required by LegacyInputIterator)
// so that we don't promise always returning CodeUnits.
class Proxy {
public:
explicit Proxy(CodeUnits<CP32, UnitIter> &units) : units_(units) {}
CodeUnits<CP32, UnitIter> &operator*() { return units_; }
CodeUnits<CP32, UnitIter> *operator->() { return &units_; }
private:
CodeUnits<CP32, UnitIter> units_;
};
public:
using value_type = CodeUnits<CP32, UnitIter>;
using reference = value_type;
using pointer = Proxy;
using difference_type = prv::iter_difference_t<UnitIter>;
using iterator_category = std::conditional_t<
prv::bidirectional_iterator<UnitIter>,
std::bidirectional_iterator_tag,
std::forward_iterator_tag>;
U_FORCE_INLINE UTFIterator(UnitIter start, UnitIter p, LimitIter limit) :
p_(p), start_(start), limit_(limit), units_(0, 0, false, p, p) {}
U_FORCE_INLINE UTFIterator(UnitIter p, LimitIter limit) :
p_(p), start_(p), limit_(limit), units_(0, 0, false, p, p) {}
U_FORCE_INLINE explicit UTFIterator(UnitIter p) : p_(p), start_(p), limit_(p), units_(0, 0, false, p, p) {}
U_FORCE_INLINE UTFIterator() : p_{}, start_{}, limit_{}, units_(0, 0, false, p_, p_) {}
U_FORCE_INLINE UTFIterator(UTFIterator &&src) noexcept = default;
U_FORCE_INLINE UTFIterator &operator=(UTFIterator &&src) noexcept = default;
U_FORCE_INLINE UTFIterator(const UTFIterator &other) = default;
U_FORCE_INLINE UTFIterator &operator=(const UTFIterator &other) = default;
U_FORCE_INLINE bool operator==(const UTFIterator &other) const {
return getLogicalPosition() == other.getLogicalPosition();
}
U_FORCE_INLINE bool operator!=(const UTFIterator &other) const { return !operator==(other); }
// Asymmetric equality & nonequality with a sentinel type.
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator==(const UTFIterator &iter, const Sentinel &s) {
return iter.getLogicalPosition() == s;
}
#if U_CPLUSPLUS_VERSION < 20
// C++17: Need to define all four combinations of == / != vs. parameter order.
// Once we require C++20, we could remove all but the first == because
// the compiler would generate the rest.
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator==(const Sentinel &s, const UTFIterator &iter) {
return iter.getLogicalPosition() == s;
}
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator!=(const UTFIterator &iter, const Sentinel &s) { return !(iter == s); }
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator!=(const Sentinel &s, const UTFIterator &iter) { return !(iter == s); }
#endif // C++17
U_FORCE_INLINE CodeUnits<CP32, UnitIter> operator*() const {
if (state_ == 0) {
UnitIter p0 = p_;
units_ = Impl::readAndInc(p0, p_, limit_);
state_ = 1;
}
return units_;
}
U_FORCE_INLINE Proxy operator->() const {
if (state_ == 0) {
UnitIter p0 = p_;
units_ = Impl::readAndInc(p0, p_, limit_);
state_ = 1;
}
return Proxy(units_);
}
U_FORCE_INLINE UTFIterator &operator++() { // pre-increment
if (state_ > 0) {
// operator*() called readAndInc() so p_ is already ahead.
state_ = 0;
} else if (state_ == 0) {
Impl::inc(p_, limit_);
} else /* state_ < 0 */ {
// operator--() called decAndRead() so we know how far to skip.
p_ = units_.end();
state_ = 0;
}
return *this;
}
U_FORCE_INLINE UTFIterator operator++(int) { // post-increment
if (state_ > 0) {
// operator*() called readAndInc() so p_ is already ahead.
UTFIterator result(*this);
state_ = 0;
return result;
} else if (state_ == 0) {
UnitIter p0 = p_;
units_ = Impl::readAndInc(p0, p_, limit_);
UTFIterator result(*this);
result.state_ = 1;
// keep this->state_ == 0
return result;
} else /* state_ < 0 */ {
UTFIterator result(*this);
// operator--() called decAndRead() so we know how far to skip.
p_ = units_.end();
state_ = 0;
return result;
}
}
template<typename Iter = UnitIter>
U_FORCE_INLINE
std::enable_if_t<prv::bidirectional_iterator<Iter>, UTFIterator &>
operator--() { // pre-decrement
if (state_ > 0) {
// operator*() called readAndInc() so p_ is ahead of the logical position.
p_ = units_.begin();
}
units_ = Impl::decAndRead(start_, p_);
state_ = -1;
return *this;
}
template<typename Iter = UnitIter>
U_FORCE_INLINE
std::enable_if_t<prv::bidirectional_iterator<Iter>, UTFIterator>
operator--(int) { // post-decrement
UTFIterator result(*this);
operator--();
return result;
}
private:
friend class std::reverse_iterator<UTFIterator<CP32, behavior, UnitIter>>;
U_FORCE_INLINE UnitIter getLogicalPosition() const {
return state_ <= 0 ? p_ : units_.begin();
}
// operator*() etc. are logically const.
mutable UnitIter p_;
// In a validating iterator, we need start_ & limit_ so that when we read a code point
// (forward or backward) we can test if there are enough code units.
UnitIter start_;
LimitIter limit_;
// Keep state so that we call readAndInc() only once for both operator*() and ++
// to make it easy for the compiler to optimize.
mutable CodeUnits<CP32, UnitIter> units_;
// >0: units_ = readAndInc(), p_ = units limit
// which means that p_ is ahead of its logical position
// 0: initial state
// <0: units_ = decAndRead(), p_ = units start
mutable int8_t state_ = 0;
};
#ifndef U_IN_DOXYGEN
// Partial template specialization for single-pass input iterator.
template<typename CP32, UTFIllFormedBehavior behavior, typename UnitIter, typename LimitIter>
class UTFIterator<
CP32, behavior,
UnitIter, LimitIter,
std::enable_if_t<!prv::forward_iterator<UnitIter>>> {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
using Impl = UTFImpl<CP32, behavior, UnitIter, LimitIter>;
// Proxy type for post-increment return value, to make *iter++ work.
// Also for operator->() (required by LegacyInputIterator)
// so that we don't promise always returning CodeUnits.
class Proxy {
public:
explicit Proxy(CodeUnits<CP32, UnitIter> &units) : units_(units) {}
CodeUnits<CP32, UnitIter> &operator*() { return units_; }
CodeUnits<CP32, UnitIter> *operator->() { return &units_; }
private:
CodeUnits<CP32, UnitIter> units_;
};
public:
using value_type = CodeUnits<CP32, UnitIter>;
using reference = value_type;
using pointer = Proxy;
using difference_type = prv::iter_difference_t<UnitIter>;
using iterator_category = std::input_iterator_tag;
U_FORCE_INLINE UTFIterator(UnitIter p, LimitIter limit) : p_(std::move(p)), limit_(std::move(limit)) {}
// Constructs an iterator start or limit sentinel.
// Requires p to be copyable.
U_FORCE_INLINE explicit UTFIterator(UnitIter p) : p_(std::move(p)), limit_(p_) {}
U_FORCE_INLINE UTFIterator(UTFIterator &&src) noexcept = default;
U_FORCE_INLINE UTFIterator &operator=(UTFIterator &&src) noexcept = default;
U_FORCE_INLINE UTFIterator(const UTFIterator &other) = default;
U_FORCE_INLINE UTFIterator &operator=(const UTFIterator &other) = default;
U_FORCE_INLINE bool operator==(const UTFIterator &other) const {
return p_ == other.p_ && ahead_ == other.ahead_;
// Strictly speaking, we should check if the logical position is the same.
// However, we cannot advance, or do arithmetic with, a single-pass UnitIter.
}
U_FORCE_INLINE bool operator!=(const UTFIterator &other) const { return !operator==(other); }
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator==(const UTFIterator &iter, const Sentinel &s) {
return !iter.ahead_ && iter.p_ == s;
}
#if U_CPLUSPLUS_VERSION < 20
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator==(const Sentinel &s, const UTFIterator &iter) {
return !iter.ahead_ && iter.p_ == s;
}
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator!=(const UTFIterator &iter, const Sentinel &s) { return !(iter == s); }
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator!=(const Sentinel &s, const UTFIterator &iter) { return !(iter == s); }
#endif // C++17
U_FORCE_INLINE CodeUnits<CP32, UnitIter> operator*() const {
if (!ahead_) {
units_ = Impl::readAndInc(p_, p_, limit_);
ahead_ = true;
}
return units_;
}
U_FORCE_INLINE Proxy operator->() const {
if (!ahead_) {
units_ = Impl::readAndInc(p_, p_, limit_);
ahead_ = true;
}
return Proxy(units_);
}
U_FORCE_INLINE UTFIterator &operator++() { // pre-increment
if (ahead_) {
// operator*() called readAndInc() so p_ is already ahead.
ahead_ = false;
} else {
Impl::inc(p_, limit_);
}
return *this;
}
U_FORCE_INLINE Proxy operator++(int) { // post-increment
if (ahead_) {
// operator*() called readAndInc() so p_ is already ahead.
ahead_ = false;
} else {
units_ = Impl::readAndInc(p_, p_, limit_);
// keep this->ahead_ == false
}
return Proxy(units_);
}
private:
// operator*() etc. are logically const.
mutable UnitIter p_;
// In a validating iterator, we need limit_ so that when we read a code point
// we can test if there are enough code units.
LimitIter limit_;
// Keep state so that we call readAndInc() only once for both operator*() and ++
// so that we can use a single-pass input iterator for UnitIter.
mutable CodeUnits<CP32, UnitIter> units_ = {0, 0, false};
// true: units_ = readAndInc(), p_ = units limit
// which means that p_ is ahead of its logical position
// false: initial state
mutable bool ahead_ = false;
};
#endif // U_IN_DOXYGEN
} // namespace U_HEADER_ONLY_NAMESPACE
#ifndef U_IN_DOXYGEN
// Bespoke specialization of reverse_iterator.
// The default implementation implements reverse operator*() and ++ in a way
// that does most of the same work twice for reading variable-length sequences.
template<typename CP32, UTFIllFormedBehavior behavior, typename UnitIter>
class std::reverse_iterator<U_HEADER_ONLY_NAMESPACE::UTFIterator<CP32, behavior, UnitIter>> {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
using Impl = U_HEADER_ONLY_NAMESPACE::UTFImpl<CP32, behavior, UnitIter>;
using CodeUnits_ = U_HEADER_ONLY_NAMESPACE::CodeUnits<CP32, UnitIter>;
// Proxy type for operator->() (required by LegacyInputIterator)
// so that we don't promise always returning CodeUnits.
class Proxy {
public:
explicit Proxy(CodeUnits_ units) : units_(units) {}
CodeUnits_ &operator*() { return units_; }
CodeUnits_ *operator->() { return &units_; }
private:
CodeUnits_ units_;
};
public:
using value_type = CodeUnits_;
using reference = value_type;
using pointer = Proxy;
using difference_type = U_HEADER_ONLY_NAMESPACE::prv::iter_difference_t<UnitIter>;
using iterator_category = std::bidirectional_iterator_tag;
U_FORCE_INLINE explicit reverse_iterator(U_HEADER_ONLY_NAMESPACE::UTFIterator<CP32, behavior, UnitIter> iter) :
p_(iter.getLogicalPosition()), start_(iter.start_), limit_(iter.limit_),
units_(0, 0, false, p_, p_) {}
U_FORCE_INLINE reverse_iterator() : p_{}, start_{}, limit_{}, units_(0, 0, false, p_, p_) {}
U_FORCE_INLINE reverse_iterator(reverse_iterator &&src) noexcept = default;
U_FORCE_INLINE reverse_iterator &operator=(reverse_iterator &&src) noexcept = default;
U_FORCE_INLINE reverse_iterator(const reverse_iterator &other) = default;
U_FORCE_INLINE reverse_iterator &operator=(const reverse_iterator &other) = default;
U_FORCE_INLINE bool operator==(const reverse_iterator &other) const {
return getLogicalPosition() == other.getLogicalPosition();
}
U_FORCE_INLINE bool operator!=(const reverse_iterator &other) const { return !operator==(other); }
U_FORCE_INLINE CodeUnits_ operator*() const {
if (state_ == 0) {
units_ = Impl::decAndRead(start_, p_);
state_ = -1;
}
return units_;
}
U_FORCE_INLINE Proxy operator->() const {
if (state_ == 0) {
units_ = Impl::decAndRead(start_, p_);
state_ = -1;
}
return Proxy(units_);
}
U_FORCE_INLINE reverse_iterator &operator++() { // pre-increment
if (state_ < 0) {
// operator*() called decAndRead() so p_ is already behind.
state_ = 0;
} else if (state_ == 0) {
Impl::dec(start_, p_);
} else /* state_ > 0 */ {
// operator--() called readAndInc() so we know how far to skip.
p_ = units_.begin();
state_ = 0;
}
return *this;
}
U_FORCE_INLINE reverse_iterator operator++(int) { // post-increment
if (state_ < 0) {
// operator*() called decAndRead() so p_ is already behind.
reverse_iterator result(*this);
state_ = 0;
return result;
} else if (state_ == 0) {
units_ = Impl::decAndRead(start_, p_);
reverse_iterator result(*this);
result.state_ = -1;
// keep this->state_ == 0
return result;
} else /* state_ > 0 */ {
reverse_iterator result(*this);
// operator--() called readAndInc() so we know how far to skip.
p_ = units_.begin();
state_ = 0;
return result;
}
}
U_FORCE_INLINE reverse_iterator &operator--() { // pre-decrement
if (state_ < 0) {
// operator*() called decAndRead() so p_ is behind the logical position.
p_ = units_.end();
}
UnitIter p0 = p_;
units_ = Impl::readAndInc(p0, p_, limit_);
state_ = 1;
return *this;
}
U_FORCE_INLINE reverse_iterator operator--(int) { // post-decrement
reverse_iterator result(*this);
operator--();
return result;
}
private:
U_FORCE_INLINE UnitIter getLogicalPosition() const {
return state_ >= 0 ? p_ : units_.end();
}
// operator*() etc. are logically const.
mutable UnitIter p_;
// In a validating iterator, we need start_ & limit_ so that when we read a code point
// (forward or backward) we can test if there are enough code units.
UnitIter start_;
UnitIter limit_;
// Keep state so that we call decAndRead() only once for both operator*() and ++
// to make it easy for the compiler to optimize.
mutable CodeUnits_ units_;
// >0: units_ = readAndInc(), p_ = units limit
// 0: initial state
// <0: units_ = decAndRead(), p_ = units start
// which means that p_ is behind its logical position
mutable int8_t state_ = 0;
};
#endif // U_IN_DOXYGEN
namespace U_HEADER_ONLY_NAMESPACE {
template<typename CP32, UTFIllFormedBehavior behavior,
typename UnitIter, typename LimitIter = UnitIter>
auto utfIterator(UnitIter start, UnitIter p, LimitIter limit) {
return UTFIterator<CP32, behavior, UnitIter, LimitIter>(
std::move(start), std::move(p), std::move(limit));
}
template<typename CP32, UTFIllFormedBehavior behavior,
typename UnitIter, typename LimitIter = UnitIter>
auto utfIterator(UnitIter p, LimitIter limit) {
return UTFIterator<CP32, behavior, UnitIter, LimitIter>(
std::move(p), std::move(limit));
}
// Note: We should only enable the following factory function for a copyable UnitIter.
// In C++17, we would have to partially specialize with enable_if_t testing for forward_iterator,
// but a function template partial specialization is not allowed.
// In C++20, we might be able to require the std::copyable concept.
template<typename CP32, UTFIllFormedBehavior behavior, typename UnitIter>
auto utfIterator(UnitIter p) {
return UTFIterator<CP32, behavior, UnitIter>(std::move(p));
}
template<typename CP32, UTFIllFormedBehavior behavior, typename Range>
class UTFStringCodePoints {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
public:
UTFStringCodePoints() = default;
template<typename R = Range, typename = std::enable_if_t<!std::is_reference_v<R>>>
explicit UTFStringCodePoints(Range unitRange) : unitRange(std::move(unitRange)) {}
template<typename R = Range, typename = std::enable_if_t<std::is_reference_v<R>>, typename = void>
explicit UTFStringCodePoints(Range unitRange) : unitRange(unitRange) {}
UTFStringCodePoints(const UTFStringCodePoints &other) = default;
UTFStringCodePoints &operator=(const UTFStringCodePoints &other) = default;
auto begin() {
return utfIterator<CP32, behavior>(unitRange.begin(), unitRange.end());
}
template<typename R = Range, typename = std::enable_if_t<prv::range<const R>>>
auto begin() const {
return utfIterator<CP32, behavior>(unitRange.begin(), unitRange.end());
}
auto end() {
using UnitIter = decltype(unitRange.begin());
using LimitIter = decltype(unitRange.end());
if constexpr (!std::is_same_v<UnitIter, LimitIter>) {
// Return the code unit sentinel.
return unitRange.end();
} else if constexpr (prv::bidirectional_iterator<UnitIter>) {
return utfIterator<CP32, behavior>(unitRange.begin(), unitRange.end(), unitRange.end());
} else {
// The input iterator specialization has no three-argument constructor.
return utfIterator<CP32, behavior>(unitRange.end(), unitRange.end());
}
}
template<typename R = Range, typename = std::enable_if_t<prv::range<const R>>>
auto end() const {
using UnitIter = decltype(unitRange.begin());
using LimitIter = decltype(unitRange.end());
if constexpr (!std::is_same_v<UnitIter, LimitIter>) {
// Return the code unit sentinel.
return unitRange.end();
} else if constexpr (prv::bidirectional_iterator<UnitIter>) {
return utfIterator<CP32, behavior>(unitRange.begin(), unitRange.end(), unitRange.end());
} else {
// The input iterator specialization has no three-argument constructor.
return utfIterator<CP32, behavior>(unitRange.end(), unitRange.end());
}
}
auto rbegin() const {
return std::make_reverse_iterator(end());
}
auto rend() const {
return std::make_reverse_iterator(begin());
}
private:
Range unitRange;
};
template<typename CP32, UTFIllFormedBehavior behavior>
struct UTFStringCodePointsAdaptor
#if U_CPLUSPLUS_VERSION >= 23 && __cpp_lib_ranges >= 2022'02 && \
__cpp_lib_bind_back >= 2022'02 // http://wg21.link/P2387R3.
: std::ranges::range_adaptor_closure<UTFStringCodePointsAdaptor<CP32, behavior>>
#endif
{
template<typename Range>
auto operator()(Range &&unitRange) const {
#if defined(__cpp_lib_ranges) && __cpp_lib_ranges >= 2021'10 // We need https://wg21.link/P2415R2.
return UTFStringCodePoints<CP32, behavior, std::ranges::views::all_t<Range>>(
std::forward<Range>(unitRange));
#else
if constexpr (prv::is_basic_string_view_v<std::decay_t<Range>>) {
// Take basic_string_view by copy, not by reference. In C++20 this is handled by
// all_t<Range>, which is Range if Range is a view.
return UTFStringCodePoints<CP32, behavior, std::decay_t<Range>>(
std::forward<Range>(unitRange));
} else {
return UTFStringCodePoints<CP32, behavior, Range>(std::forward<Range>(unitRange));
}
#endif
}
};
template<typename CP32, UTFIllFormedBehavior behavior>
constexpr UTFStringCodePointsAdaptor<CP32, behavior> utfStringCodePoints;
// Non-validating iterators ------------------------------------------------ ***
template<typename CP32, typename UnitIter, typename = void>
class UnsafeUTFIterator {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
using Impl = UnsafeUTFImpl<CP32, UnitIter>;
// Proxy type for operator->() (required by LegacyInputIterator)
// so that we don't promise always returning UnsafeCodeUnits.
class Proxy {
public:
explicit Proxy(UnsafeCodeUnits<CP32, UnitIter> &units) : units_(units) {}
UnsafeCodeUnits<CP32, UnitIter> &operator*() { return units_; }
UnsafeCodeUnits<CP32, UnitIter> *operator->() { return &units_; }
private:
UnsafeCodeUnits<CP32, UnitIter> units_;
};
public:
using value_type = UnsafeCodeUnits<CP32, UnitIter>;
using reference = value_type;
using pointer = Proxy;
using difference_type = prv::iter_difference_t<UnitIter>;
using iterator_category = std::conditional_t<
prv::bidirectional_iterator<UnitIter>,
std::bidirectional_iterator_tag,
std::forward_iterator_tag>;
U_FORCE_INLINE explicit UnsafeUTFIterator(UnitIter p) : p_(p), units_(0, 0, p, p) {}
U_FORCE_INLINE UnsafeUTFIterator() : p_{}, units_(0, 0, p_, p_) {}
U_FORCE_INLINE UnsafeUTFIterator(UnsafeUTFIterator &&src) noexcept = default;
U_FORCE_INLINE UnsafeUTFIterator &operator=(UnsafeUTFIterator &&src) noexcept = default;
U_FORCE_INLINE UnsafeUTFIterator(const UnsafeUTFIterator &other) = default;
U_FORCE_INLINE UnsafeUTFIterator &operator=(const UnsafeUTFIterator &other) = default;
U_FORCE_INLINE bool operator==(const UnsafeUTFIterator &other) const {
return getLogicalPosition() == other.getLogicalPosition();
}
U_FORCE_INLINE bool operator!=(const UnsafeUTFIterator &other) const { return !operator==(other); }
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator==(const UnsafeUTFIterator &iter, const Sentinel &s) {
return iter.getLogicalPosition() == s;
}
#if U_CPLUSPLUS_VERSION < 20
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator==(const Sentinel &s, const UnsafeUTFIterator &iter) {
return iter.getLogicalPosition() == s;
}
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator!=(const UnsafeUTFIterator &iter, const Sentinel &s) { return !(iter == s); }
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator!=(const Sentinel &s, const UnsafeUTFIterator &iter) { return !(iter == s); }
#endif // C++17
U_FORCE_INLINE UnsafeCodeUnits<CP32, UnitIter> operator*() const {
if (state_ == 0) {
UnitIter p0 = p_;
units_ = Impl::readAndInc(p0, p_);
state_ = 1;
}
return units_;
}
U_FORCE_INLINE Proxy operator->() const {
if (state_ == 0) {
UnitIter p0 = p_;
units_ = Impl::readAndInc(p0, p_);
state_ = 1;
}
return Proxy(units_);
}
U_FORCE_INLINE UnsafeUTFIterator &operator++() { // pre-increment
if (state_ > 0) {
// operator*() called readAndInc() so p_ is already ahead.
state_ = 0;
} else if (state_ == 0) {
Impl::inc(p_);
} else /* state_ < 0 */ {
// operator--() called decAndRead() so we know how far to skip.
p_ = units_.end();
state_ = 0;
}
return *this;
}
U_FORCE_INLINE UnsafeUTFIterator operator++(int) { // post-increment
if (state_ > 0) {
// operator*() called readAndInc() so p_ is already ahead.
UnsafeUTFIterator result(*this);
state_ = 0;
return result;
} else if (state_ == 0) {
UnitIter p0 = p_;
units_ = Impl::readAndInc(p0, p_);
UnsafeUTFIterator result(*this);
result.state_ = 1;
// keep this->state_ == 0
return result;
} else /* state_ < 0 */ {
UnsafeUTFIterator result(*this);
// operator--() called decAndRead() so we know how far to skip.
p_ = units_.end();
state_ = 0;
return result;
}
}
template<typename Iter = UnitIter>
U_FORCE_INLINE
std::enable_if_t<prv::bidirectional_iterator<Iter>, UnsafeUTFIterator &>
operator--() { // pre-decrement
if (state_ > 0) {
// operator*() called readAndInc() so p_ is ahead of the logical position.
p_ = units_.begin();
}
units_ = Impl::decAndRead(p_);
state_ = -1;
return *this;
}
template<typename Iter = UnitIter>
U_FORCE_INLINE
std::enable_if_t<prv::bidirectional_iterator<Iter>, UnsafeUTFIterator>
operator--(int) { // post-decrement
UnsafeUTFIterator result(*this);
operator--();
return result;
}
private:
friend class std::reverse_iterator<UnsafeUTFIterator<CP32, UnitIter>>;
U_FORCE_INLINE UnitIter getLogicalPosition() const {
return state_ <= 0 ? p_ : units_.begin();
}
// operator*() etc. are logically const.
mutable UnitIter p_;
// Keep state so that we call readAndInc() only once for both operator*() and ++
// to make it easy for the compiler to optimize.
mutable UnsafeCodeUnits<CP32, UnitIter> units_;
// >0: units_ = readAndInc(), p_ = units limit
// which means that p_ is ahead of its logical position
// 0: initial state
// <0: units_ = decAndRead(), p_ = units start
mutable int8_t state_ = 0;
};
#ifndef U_IN_DOXYGEN
// Partial template specialization for single-pass input iterator.
template<typename CP32, typename UnitIter>
class UnsafeUTFIterator<
CP32,
UnitIter,
std::enable_if_t<!prv::forward_iterator<UnitIter>>> {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
using Impl = UnsafeUTFImpl<CP32, UnitIter>;
// Proxy type for post-increment return value, to make *iter++ work.
// Also for operator->() (required by LegacyInputIterator)
// so that we don't promise always returning UnsafeCodeUnits.
class Proxy {
public:
explicit Proxy(UnsafeCodeUnits<CP32, UnitIter> &units) : units_(units) {}
UnsafeCodeUnits<CP32, UnitIter> &operator*() { return units_; }
UnsafeCodeUnits<CP32, UnitIter> *operator->() { return &units_; }
private:
UnsafeCodeUnits<CP32, UnitIter> units_;
};
public:
using value_type = UnsafeCodeUnits<CP32, UnitIter>;
using reference = value_type;
using pointer = Proxy;
using difference_type = prv::iter_difference_t<UnitIter>;
using iterator_category = std::input_iterator_tag;
U_FORCE_INLINE explicit UnsafeUTFIterator(UnitIter p) : p_(std::move(p)) {}
U_FORCE_INLINE UnsafeUTFIterator(UnsafeUTFIterator &&src) noexcept = default;
U_FORCE_INLINE UnsafeUTFIterator &operator=(UnsafeUTFIterator &&src) noexcept = default;
U_FORCE_INLINE UnsafeUTFIterator(const UnsafeUTFIterator &other) = default;
U_FORCE_INLINE UnsafeUTFIterator &operator=(const UnsafeUTFIterator &other) = default;
U_FORCE_INLINE bool operator==(const UnsafeUTFIterator &other) const {
return p_ == other.p_ && ahead_ == other.ahead_;
// Strictly speaking, we should check if the logical position is the same.
// However, we cannot advance, or do arithmetic with, a single-pass UnitIter.
}
U_FORCE_INLINE bool operator!=(const UnsafeUTFIterator &other) const { return !operator==(other); }
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator==(const UnsafeUTFIterator &iter, const Sentinel &s) {
return !iter.ahead_ && iter.p_ == s;
}
#if U_CPLUSPLUS_VERSION < 20
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator==(const Sentinel &s, const UnsafeUTFIterator &iter) {
return !iter.ahead_ && iter.p_ == s;
}
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator!=(const UnsafeUTFIterator &iter, const Sentinel &s) { return !(iter == s); }
template<typename Sentinel> U_FORCE_INLINE friend
std::enable_if_t<
!std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
bool>
operator!=(const Sentinel &s, const UnsafeUTFIterator &iter) { return !(iter == s); }
#endif // C++17
U_FORCE_INLINE UnsafeCodeUnits<CP32, UnitIter> operator*() const {
if (!ahead_) {
units_ = Impl::readAndInc(p_, p_);
ahead_ = true;
}
return units_;
}
U_FORCE_INLINE Proxy operator->() const {
if (!ahead_) {
units_ = Impl::readAndInc(p_, p_);
ahead_ = true;
}
return Proxy(units_);
}
U_FORCE_INLINE UnsafeUTFIterator &operator++() { // pre-increment
if (ahead_) {
// operator*() called readAndInc() so p_ is already ahead.
ahead_ = false;
} else {
Impl::inc(p_);
}
return *this;
}
U_FORCE_INLINE Proxy operator++(int) { // post-increment
if (ahead_) {
// operator*() called readAndInc() so p_ is already ahead.
ahead_ = false;
} else {
units_ = Impl::readAndInc(p_, p_);
// keep this->ahead_ == false
}
return Proxy(units_);
}
private:
// operator*() etc. are logically const.
mutable UnitIter p_;
// Keep state so that we call readAndInc() only once for both operator*() and ++
// so that we can use a single-pass input iterator for UnitIter.
mutable UnsafeCodeUnits<CP32, UnitIter> units_ = {0, 0};
// true: units_ = readAndInc(), p_ = units limit
// which means that p_ is ahead of its logical position
// false: initial state
mutable bool ahead_ = false;
};
#endif // U_IN_DOXYGEN
} // namespace U_HEADER_ONLY_NAMESPACE
#ifndef U_IN_DOXYGEN
// Bespoke specialization of reverse_iterator.
// The default implementation implements reverse operator*() and ++ in a way
// that does most of the same work twice for reading variable-length sequences.
template<typename CP32, typename UnitIter>
class std::reverse_iterator<U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator<CP32, UnitIter>> {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
using Impl = U_HEADER_ONLY_NAMESPACE::UnsafeUTFImpl<CP32, UnitIter>;
using UnsafeCodeUnits_ = U_HEADER_ONLY_NAMESPACE::UnsafeCodeUnits<CP32, UnitIter>;
// Proxy type for operator->() (required by LegacyInputIterator)
// so that we don't promise always returning UnsafeCodeUnits.
class Proxy {
public:
explicit Proxy(UnsafeCodeUnits_ units) : units_(units) {}
UnsafeCodeUnits_ &operator*() { return units_; }
UnsafeCodeUnits_ *operator->() { return &units_; }
private:
UnsafeCodeUnits_ units_;
};
public:
using value_type = UnsafeCodeUnits_;
using reference = value_type;
using pointer = Proxy;
using difference_type = U_HEADER_ONLY_NAMESPACE::prv::iter_difference_t<UnitIter>;
using iterator_category = std::bidirectional_iterator_tag;
U_FORCE_INLINE explicit reverse_iterator(U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator<CP32, UnitIter> iter) :
p_(iter.getLogicalPosition()), units_(0, 0, p_, p_) {}
U_FORCE_INLINE reverse_iterator() : p_{}, units_(0, 0, p_, p_) {}
U_FORCE_INLINE reverse_iterator(reverse_iterator &&src) noexcept = default;
U_FORCE_INLINE reverse_iterator &operator=(reverse_iterator &&src) noexcept = default;
U_FORCE_INLINE reverse_iterator(const reverse_iterator &other) = default;
U_FORCE_INLINE reverse_iterator &operator=(const reverse_iterator &other) = default;
U_FORCE_INLINE bool operator==(const reverse_iterator &other) const {
return getLogicalPosition() == other.getLogicalPosition();
}
U_FORCE_INLINE bool operator!=(const reverse_iterator &other) const { return !operator==(other); }
U_FORCE_INLINE UnsafeCodeUnits_ operator*() const {
if (state_ == 0) {
units_ = Impl::decAndRead(p_);
state_ = -1;
}
return units_;
}
U_FORCE_INLINE Proxy operator->() const {
if (state_ == 0) {
units_ = Impl::decAndRead(p_);
state_ = -1;
}
return Proxy(units_);
}
U_FORCE_INLINE reverse_iterator &operator++() { // pre-increment
if (state_ < 0) {
// operator*() called decAndRead() so p_ is already behind.
state_ = 0;
} else if (state_ == 0) {
Impl::dec(p_);
} else /* state_ > 0 */ {
// operator--() called readAndInc() so we know how far to skip.
p_ = units_.begin();
state_ = 0;
}
return *this;
}
U_FORCE_INLINE reverse_iterator operator++(int) { // post-increment
if (state_ < 0) {
// operator*() called decAndRead() so p_ is already behind.
reverse_iterator result(*this);
state_ = 0;
return result;
} else if (state_ == 0) {
units_ = Impl::decAndRead(p_);
reverse_iterator result(*this);
result.state_ = -1;
// keep this->state_ == 0
return result;
} else /* state_ > 0 */ {
reverse_iterator result(*this);
// operator--() called readAndInc() so we know how far to skip.
p_ = units_.begin();
state_ = 0;
return result;
}
}
U_FORCE_INLINE reverse_iterator &operator--() { // pre-decrement
if (state_ < 0) {
// operator*() called decAndRead() so p_ is behind the logical position.
p_ = units_.end();
}
UnitIter p0 = p_;
units_ = Impl::readAndInc(p0, p_);
state_ = 1;
return *this;
}
U_FORCE_INLINE reverse_iterator operator--(int) { // post-decrement
reverse_iterator result(*this);
operator--();
return result;
}
private:
U_FORCE_INLINE UnitIter getLogicalPosition() const {
return state_ >= 0 ? p_ : units_.end();
}
// operator*() etc. are logically const.
mutable UnitIter p_;
// Keep state so that we call decAndRead() only once for both operator*() and ++
// to make it easy for the compiler to optimize.
mutable UnsafeCodeUnits_ units_;
// >0: units_ = readAndInc(), p_ = units limit
// 0: initial state
// <0: units_ = decAndRead(), p_ = units start
// which means that p_ is behind its logical position
mutable int8_t state_ = 0;
};
#endif // U_IN_DOXYGEN
namespace U_HEADER_ONLY_NAMESPACE {
template<typename CP32, typename UnitIter>
auto unsafeUTFIterator(UnitIter iter) {
return UnsafeUTFIterator<CP32, UnitIter>(std::move(iter));
}
template<typename CP32, typename Range>
class UnsafeUTFStringCodePoints {
static_assert(sizeof(CP32) == 4, "CP32 must be a 32-bit type to hold a code point");
public:
UnsafeUTFStringCodePoints() = default;
template<typename R = Range, typename = std::enable_if_t<!std::is_reference_v<R>>>
explicit UnsafeUTFStringCodePoints(Range unitRange) : unitRange(std::move(unitRange)) {}
template<typename R = Range, typename = std::enable_if_t<std::is_reference_v<R>>, typename = void>
explicit UnsafeUTFStringCodePoints(Range unitRange) : unitRange(unitRange) {}
UnsafeUTFStringCodePoints(const UnsafeUTFStringCodePoints &other) = default;
UnsafeUTFStringCodePoints &operator=(const UnsafeUTFStringCodePoints &other) = default;
auto begin() {
return unsafeUTFIterator<CP32>(unitRange.begin());
}
template<typename R = Range, typename = std::enable_if_t<prv::range<const R>>>
auto begin() const {
return unsafeUTFIterator<CP32>(unitRange.begin());
}
auto end() {
using UnitIter = decltype(unitRange.begin());
using LimitIter = decltype(unitRange.end());
if constexpr (!std::is_same_v<UnitIter, LimitIter>) {
// Return the code unit sentinel.
return unitRange.end();
} else {
return unsafeUTFIterator<CP32>(unitRange.end());
}
}
template<typename R = Range, typename = std::enable_if_t<prv::range<const R>>>
auto end() const {
using UnitIter = decltype(unitRange.begin());
using LimitIter = decltype(unitRange.end());
if constexpr (!std::is_same_v<UnitIter, LimitIter>) {
// Return the code unit sentinel.
return unitRange.end();
} else {
return unsafeUTFIterator<CP32>(unitRange.end());
}
}
auto rbegin() const {
return std::make_reverse_iterator(end());
}
auto rend() const {
return std::make_reverse_iterator(begin());
}
private:
Range unitRange;
};
template<typename CP32>
struct UnsafeUTFStringCodePointsAdaptor
#if U_CPLUSPLUS_VERSION >= 23 && __cpp_lib_ranges >= 2022'02 && \
__cpp_lib_bind_back >= 2022'02 // http://wg21.link/P2387R3.
: std::ranges::range_adaptor_closure<UnsafeUTFStringCodePointsAdaptor<CP32>>
#endif
{
template<typename Range>
auto operator()(Range &&unitRange) const {
#if defined(__cpp_lib_ranges) && __cpp_lib_ranges >= 2021'10 // We need https://wg21.link/P2415R2.
return UnsafeUTFStringCodePoints<CP32, std::ranges::views::all_t<Range>>(std::forward<Range>(unitRange));
#else
if constexpr (prv::is_basic_string_view_v<std::decay_t<Range>>) {
// Take basic_string_view by copy, not by reference. In C++20 this is handled by
// all_t<Range>, which is Range if Range is a view.
return UnsafeUTFStringCodePoints<CP32, std::decay_t<Range>>(std::forward<Range>(unitRange));
} else {
return UnsafeUTFStringCodePoints<CP32, Range>(std::forward<Range>(unitRange));
}
#endif
}
};
template<typename CP32>
constexpr UnsafeUTFStringCodePointsAdaptor<CP32> unsafeUTFStringCodePoints;
} // namespace U_HEADER_ONLY_NAMESPACE
#if defined(__cpp_lib_ranges)
template <typename CP32, UTFIllFormedBehavior behavior, typename Range>
constexpr bool std::ranges::enable_borrowed_range<
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints<CP32, behavior, Range>> =
std::ranges::enable_borrowed_range<Range>;
template <typename CP32, typename Range>
constexpr bool std::ranges::enable_borrowed_range<
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints<CP32, Range>> =
std::ranges::enable_borrowed_range<Range>;
#endif
#endif // U_HIDE_DRAFT_API
#endif // U_SHOW_CPLUSPLUS_API || U_SHOW_CPLUSPLUS_HEADER_API
#endif // __UTFITERATOR_H__
U_HEADER_ONLY_NAMESPACE::AllCodePoints
A C++ "range" over all Unicode code points U+0000..U+10FFFF.
Definition: utfiterator.h:302
U_HEADER_ONLY_NAMESPACE::AllCodePoints::begin
auto begin() const
Definition: utfiterator.h:312
U_HEADER_ONLY_NAMESPACE::AllCodePoints::AllCodePoints
AllCodePoints()
Constructor.
Definition: utfiterator.h:306
U_HEADER_ONLY_NAMESPACE::AllCodePoints::end
auto end() const
Definition: utfiterator.h:317
U_HEADER_ONLY_NAMESPACE::AllScalarValues
A C++ "range" over all Unicode scalar values U+0000..U+D7FF & U+E000..U+10FFFF.
Definition: utfiterator.h:333
U_HEADER_ONLY_NAMESPACE::AllScalarValues::begin
auto begin() const
Definition: utfiterator.h:343
U_HEADER_ONLY_NAMESPACE::AllScalarValues::AllScalarValues
AllScalarValues()
Constructor.
Definition: utfiterator.h:337
U_HEADER_ONLY_NAMESPACE::AllScalarValues::end
auto end() const
Definition: utfiterator.h:348
U_HEADER_ONLY_NAMESPACE::CodeUnits
Result of validating and decoding a code unit sequence for one code point.
Definition: utfiterator.h:487
U_HEADER_ONLY_NAMESPACE::CodeUnits::wellFormed
bool wellFormed() const
Definition: utfiterator.h:502
U_HEADER_ONLY_NAMESPACE::CodeUnits::operator=
CodeUnits & operator=(const CodeUnits &other)=default
Copy assignment operator.
U_HEADER_ONLY_NAMESPACE::CodeUnits::CodeUnits
CodeUnits(const CodeUnits &other)=default
Copy constructor.
U_HEADER_ONLY_NAMESPACE::CodeUnits::CodeUnits
CodeUnits(CP32 codePoint, uint8_t length, bool wellFormed, UnitIter start, UnitIter limit)
Definition: utfiterator.h:490
U_HEADER_ONLY_NAMESPACE::UTFIterator
Validating iterator over the code points in a Unicode string.
Definition: utfiterator.h:1094
U_HEADER_ONLY_NAMESPACE::UTFIterator::UTFIterator
U_FORCE_INLINE UTFIterator()
Default constructor.
Definition: utfiterator.h:1169
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator==
U_FORCE_INLINE friend std::enable_if_t< !std::is_same_v< Sentinel, UTFIterator > &&!std::is_same_v< Sentinel, UnitIter >, bool > operator==(const Sentinel &s, const UTFIterator &iter)
Definition: utfiterator.h:1227
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator--
U_FORCE_INLINE std::enable_if_t< prv::bidirectional_iterator< Iter >, UTFIterator > operator--(int)
Post-decrement operator.
Definition: utfiterator.h:1366
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator->
U_FORCE_INLINE Proxy operator->() const
Decodes the code unit sequence at the current position.
Definition: utfiterator.h:1277
U_HEADER_ONLY_NAMESPACE::UTFIterator::reference
value_type reference
C++ iterator boilerplate.
Definition: utfiterator.h:1113
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator=
U_FORCE_INLINE UTFIterator & operator=(UTFIterator &&src) noexcept=default
Move assignment operator.
U_HEADER_ONLY_NAMESPACE::UTFIterator::value_type
CodeUnits< CP32, UnitIter > value_type
C++ iterator boilerplate.
Definition: utfiterator.h:1111
U_HEADER_ONLY_NAMESPACE::UTFIterator::UTFIterator
U_FORCE_INLINE UTFIterator(UTFIterator &&src) noexcept=default
Move constructor.
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator!=
U_FORCE_INLINE friend std::enable_if_t< !std::is_same_v< Sentinel, UTFIterator > &&!std::is_same_v< Sentinel, UnitIter >, bool > operator!=(const UTFIterator &iter, const Sentinel &s)
Definition: utfiterator.h:1240
U_HEADER_ONLY_NAMESPACE::UTFIterator::iterator_category
std::conditional_t< prv::bidirectional_iterator< UnitIter >, std::bidirectional_iterator_tag, std::forward_iterator_tag > iterator_category
C++ iterator boilerplate.
Definition: utfiterator.h:1122
U_HEADER_ONLY_NAMESPACE::UTFIterator::UTFIterator
U_FORCE_INLINE UTFIterator(UnitIter p)
Constructs an iterator start or limit sentinel.
Definition: utfiterator.h:1163
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator!=
U_FORCE_INLINE friend std::enable_if_t< !std::is_same_v< Sentinel, UTFIterator > &&!std::is_same_v< Sentinel, UnitIter >, bool > operator!=(const Sentinel &s, const UTFIterator &iter)
Definition: utfiterator.h:1251
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator==
U_FORCE_INLINE friend std::enable_if_t< !std::is_same_v< Sentinel, UTFIterator > &&!std::is_same_v< Sentinel, UnitIter >, bool > operator==(const UTFIterator &iter, const Sentinel &s)
Definition: utfiterator.h:1208
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator--
U_FORCE_INLINE std::enable_if_t< prv::bidirectional_iterator< Iter >, UTFIterator & > operator--()
Pre-decrement operator.
Definition: utfiterator.h:1346
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator!=
U_FORCE_INLINE bool operator!=(const UTFIterator &other) const
Definition: utfiterator.h:1194
U_HEADER_ONLY_NAMESPACE::UTFIterator::pointer
Proxy pointer
C++ iterator boilerplate.
Definition: utfiterator.h:1115
U_HEADER_ONLY_NAMESPACE::UTFIterator::UTFIterator
U_FORCE_INLINE UTFIterator(UnitIter start, UnitIter p, LimitIter limit)
Constructor with start <= p < limit.
Definition: utfiterator.h:1137
U_HEADER_ONLY_NAMESPACE::UTFIterator::UTFIterator
U_FORCE_INLINE UTFIterator(const UTFIterator &other)=default
Copy constructor.
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator*
U_FORCE_INLINE CodeUnits< CP32, UnitIter > operator*() const
Decodes the code unit sequence at the current position.
Definition: utfiterator.h:1260
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator++
U_FORCE_INLINE UTFIterator operator++(int)
Post-increment operator.
Definition: utfiterator.h:1314
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator++
U_FORCE_INLINE UTFIterator & operator++()
Pre-increment operator.
Definition: utfiterator.h:1292
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator=
U_FORCE_INLINE UTFIterator & operator=(const UTFIterator &other)=default
Copy assignment operator.
U_HEADER_ONLY_NAMESPACE::UTFIterator::operator==
U_FORCE_INLINE bool operator==(const UTFIterator &other) const
Definition: utfiterator.h:1186
U_HEADER_ONLY_NAMESPACE::UTFIterator::UTFIterator
U_FORCE_INLINE UTFIterator(UnitIter p, LimitIter limit)
Constructor with start == p < limit.
Definition: utfiterator.h:1150
U_HEADER_ONLY_NAMESPACE::UTFIterator::difference_type
prv::iter_difference_t< UnitIter > difference_type
C++ iterator boilerplate.
Definition: utfiterator.h:1117
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints
A C++ "range" for validating iteration over all of the code points of a code unit range.
Definition: utfiterator.h:1777
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints::begin
auto begin() const
Definition: utfiterator.h:1823
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints::begin
auto begin()
Definition: utfiterator.h:1814
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints::UTFStringCodePoints
UTFStringCodePoints()=default
Constructs an empty C++ "range" object.
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints::rend
auto rend() const
Definition: utfiterator.h:1876
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints::rbegin
auto rbegin() const
Definition: utfiterator.h:1868
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints::operator=
UTFStringCodePoints & operator=(const UTFStringCodePoints &other)=default
Copy assignment operator.
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints::UTFStringCodePoints
UTFStringCodePoints(Range unitRange)
Constructs a C++ "range" object over the code points in the string.
Definition: utfiterator.h:1792
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints::end
auto end()
Definition: utfiterator.h:1831
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints::UTFStringCodePoints
UTFStringCodePoints(const UTFStringCodePoints &other)=default
Copy constructor.
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints::UTFStringCodePoints
UTFStringCodePoints(Range unitRange)
Constructs a C++ "range" object over the code points in the string, keeping a reference to the code u...
Definition: utfiterator.h:1802
U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints::end
auto end() const
Definition: utfiterator.h:1850
U_HEADER_ONLY_NAMESPACE::UnsafeCodeUnits
Result of decoding a code unit sequence for one code point.
Definition: utfiterator.h:367
U_HEADER_ONLY_NAMESPACE::UnsafeCodeUnits::stringView
std::enable_if_t< std::is_pointer_v< Iter >||std::is_same_v< Iter, typename std::basic_string< Unit >::iterator >||std::is_same_v< Iter, typename std::basic_string< Unit >::const_iterator >||std::is_same_v< Iter, typename std::basic_string_view< Unit >::iterator >||std::is_same_v< Iter, typename std::basic_string_view< Unit >::const_iterator >, std::basic_string_view< Unit > > stringView() const
Definition: utfiterator.h:432
U_HEADER_ONLY_NAMESPACE::UnsafeCodeUnits::begin
UnitIter begin() const
Definition: utfiterator.h:394
U_HEADER_ONLY_NAMESPACE::UnsafeCodeUnits::end
UnitIter end() const
Definition: utfiterator.h:401
U_HEADER_ONLY_NAMESPACE::UnsafeCodeUnits::codePoint
CP32 codePoint() const
Definition: utfiterator.h:387
U_HEADER_ONLY_NAMESPACE::UnsafeCodeUnits::operator=
UnsafeCodeUnits & operator=(const UnsafeCodeUnits &other)=default
Copy assignment operator.
U_HEADER_ONLY_NAMESPACE::UnsafeCodeUnits::UnsafeCodeUnits
UnsafeCodeUnits(CP32 codePoint, uint8_t length, UnitIter start, UnitIter limit)
Definition: utfiterator.h:372
U_HEADER_ONLY_NAMESPACE::UnsafeCodeUnits::length
uint8_t length() const
Definition: utfiterator.h:407
U_HEADER_ONLY_NAMESPACE::UnsafeCodeUnits::UnsafeCodeUnits
UnsafeCodeUnits(const UnsafeCodeUnits &other)=default
Copy constructor.
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator
Non-validating iterator over the code points in a Unicode string.
Definition: utfiterator.h:1952
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator--
U_FORCE_INLINE std::enable_if_t< prv::bidirectional_iterator< Iter >, UnsafeUTFIterator & > operator--()
Pre-decrement operator.
Definition: utfiterator.h:2168
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator==
U_FORCE_INLINE friend std::enable_if_t< !std::is_same_v< Sentinel, UnsafeUTFIterator > &&!std::is_same_v< Sentinel, UnitIter >, bool > operator==(const Sentinel &s, const UnsafeUTFIterator &iter)
Definition: utfiterator.h:2049
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::UnsafeUTFIterator
U_FORCE_INLINE UnsafeUTFIterator()
Default constructor.
Definition: utfiterator.h:1997
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::value_type
UnsafeCodeUnits< CP32, UnitIter > value_type
C++ iterator boilerplate.
Definition: utfiterator.h:1969
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator!=
U_FORCE_INLINE bool operator!=(const UnsafeUTFIterator &other) const
Definition: utfiterator.h:2022
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator=
U_FORCE_INLINE UnsafeUTFIterator & operator=(const UnsafeUTFIterator &other)=default
Copy assignment operator.
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::reference
value_type reference
C++ iterator boilerplate.
Definition: utfiterator.h:1971
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator!=
U_FORCE_INLINE friend std::enable_if_t< !std::is_same_v< Sentinel, UnsafeUTFIterator > &&!std::is_same_v< Sentinel, UnitIter >, bool > operator!=(const UnsafeUTFIterator &iter, const Sentinel &s)
Definition: utfiterator.h:2062
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator*
U_FORCE_INLINE UnsafeCodeUnits< CP32, UnitIter > operator*() const
Decodes the code unit sequence at the current position.
Definition: utfiterator.h:2082
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::iterator_category
std::conditional_t< prv::bidirectional_iterator< UnitIter >, std::bidirectional_iterator_tag, std::forward_iterator_tag > iterator_category
C++ iterator boilerplate.
Definition: utfiterator.h:1980
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator==
U_FORCE_INLINE friend std::enable_if_t< !std::is_same_v< Sentinel, UnsafeUTFIterator > &&!std::is_same_v< Sentinel, UnitIter >, bool > operator==(const UnsafeUTFIterator &iter, const Sentinel &s)
Definition: utfiterator.h:2034
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::pointer
Proxy pointer
C++ iterator boilerplate.
Definition: utfiterator.h:1973
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::difference_type
prv::iter_difference_t< UnitIter > difference_type
C++ iterator boilerplate.
Definition: utfiterator.h:1975
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator--
U_FORCE_INLINE std::enable_if_t< prv::bidirectional_iterator< Iter >, UnsafeUTFIterator > operator--(int)
Post-decrement operator.
Definition: utfiterator.h:2188
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator->
U_FORCE_INLINE Proxy operator->() const
Decodes the code unit sequence at the current position.
Definition: utfiterator.h:2099
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::UnsafeUTFIterator
U_FORCE_INLINE UnsafeUTFIterator(UnsafeUTFIterator &&src) noexcept=default
Move constructor.
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator++
U_FORCE_INLINE UnsafeUTFIterator operator++(int)
Post-increment operator.
Definition: utfiterator.h:2136
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::UnsafeUTFIterator
U_FORCE_INLINE UnsafeUTFIterator(const UnsafeUTFIterator &other)=default
Copy constructor.
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator++
U_FORCE_INLINE UnsafeUTFIterator & operator++()
Pre-increment operator.
Definition: utfiterator.h:2114
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::UnsafeUTFIterator
U_FORCE_INLINE UnsafeUTFIterator(UnitIter p)
Constructor; the iterator/pointer should be at a code point boundary.
Definition: utfiterator.h:1991
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator!=
U_FORCE_INLINE friend std::enable_if_t< !std::is_same_v< Sentinel, UnsafeUTFIterator > &&!std::is_same_v< Sentinel, UnitIter >, bool > operator!=(const Sentinel &s, const UnsafeUTFIterator &iter)
Definition: utfiterator.h:2073
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator==
U_FORCE_INLINE bool operator==(const UnsafeUTFIterator &other) const
Definition: utfiterator.h:2014
U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator::operator=
U_FORCE_INLINE UnsafeUTFIterator & operator=(UnsafeUTFIterator &&src) noexcept=default
Move assignment operator.
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints
A C++ "range" for non-validating iteration over all of the code points of a code unit range.
Definition: utfiterator.h:2518
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints::rend
auto rend() const
Definition: utfiterator.h:2611
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints::UnsafeUTFStringCodePoints
UnsafeUTFStringCodePoints(Range unitRange)
Constructs a C++ "range" object over the code points in the string, keeping a reference to the code u...
Definition: utfiterator.h:2543
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints::end
auto end()
Definition: utfiterator.h:2572
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints::UnsafeUTFStringCodePoints
UnsafeUTFStringCodePoints(const UnsafeUTFStringCodePoints &other)=default
Copy constructor.
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints::UnsafeUTFStringCodePoints
UnsafeUTFStringCodePoints()=default
Constructs an empty C++ "range" object.
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints::begin
auto begin() const
Definition: utfiterator.h:2564
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints::begin
auto begin()
Definition: utfiterator.h:2555
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints::UnsafeUTFStringCodePoints
UnsafeUTFStringCodePoints(Range unitRange)
Constructs a C++ "range" object over the code points in the string.
Definition: utfiterator.h:2533
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints::end
auto end() const
Definition: utfiterator.h:2588
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints::rbegin
auto rbegin() const
Definition: utfiterator.h:2603
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints::operator=
UnsafeUTFStringCodePoints & operator=(const UnsafeUTFStringCodePoints &other)=default
Copy assignment operator.
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator
Definition: utfiterator.h:248
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator::difference_type
int32_t difference_type
C++ iterator boilerplate.
Definition: utfiterator.h:258
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator::operator==
bool operator==(const CodePointsIterator &other) const
Definition: utfiterator.h:265
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator::operator!=
bool operator!=(const CodePointsIterator &other) const
Definition: utfiterator.h:267
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator::reference
value_type reference
C++ iterator boilerplate.
Definition: utfiterator.h:254
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator::operator*
CP32 operator*() const
Definition: utfiterator.h:269
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator::value_type
CP32 value_type
C++ iterator boilerplate.
Definition: utfiterator.h:252
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator::CodePointsIterator
CodePointsIterator(CP32 c)
Definition: utfiterator.h:263
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator::operator++
CodePointsIterator & operator++()
Definition: utfiterator.h:271
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator::iterator_category
std::forward_iterator_tag iterator_category
C++ iterator boilerplate.
Definition: utfiterator.h:260
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator::operator++
CodePointsIterator operator++(int)
Definition: utfiterator.h:279
U_HEADER_ONLY_NAMESPACE::prv::CodePointsIterator::pointer
CP32 * pointer
C++ iterator boilerplate.
Definition: utfiterator.h:256
icu::operator==
U_COMMON_API UBool operator==(const StringPiece &x, const StringPiece &y)
Global operator == for StringPiece.
icu::operator!=
bool operator!=(const StringPiece &x, const StringPiece &y)
Global operator != for StringPiece.
Definition: stringpiece.h:346
U_CPLUSPLUS_VERSION
#define U_CPLUSPLUS_VERSION
0 if no C++; 1, 11, 14, ...
Definition: platform.h:464
U_HEADER_ONLY_NAMESPACE::UTFStringCodePointsAdaptor
Definition: utfiterator.h:1891
U_HEADER_ONLY_NAMESPACE::UTFStringCodePointsAdaptor::operator()
auto operator()(Range &&unitRange) const
Definition: utfiterator.h:1894
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePointsAdaptor
Definition: utfiterator.h:2626
U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePointsAdaptor::operator()
auto operator()(Range &&unitRange) const
Definition: utfiterator.h:2629
U_HEADER_ONLY_NAMESPACE::prv::is_basic_string_view
Definition: utfiterator.h:237
U_HEADER_ONLY_NAMESPACE::prv::range_type
Definition: utfiterator.h:221
U_SENTINEL
#define U_SENTINEL
This value is intended for sentinel values for APIs that (take or) return single code points (UChar32...
Definition: umachine.h:469
U_FORCE_INLINE
#define U_FORCE_INLINE
Forces function inlining on compilers that are known to support it.
Definition: umachine.h:135
utf16.h
C API: 16-bit Unicode handling macros.
U16_IS_SURROGATE_TRAIL
#define U16_IS_SURROGATE_TRAIL(c)
Assuming c is a surrogate code point (U16_IS_SURROGATE(c)), is it a trail surrogate?
Definition: utf16.h:93
U16_IS_SURROGATE_LEAD
#define U16_IS_SURROGATE_LEAD(c)
Assuming c is a surrogate code point (U16_IS_SURROGATE(c)), is it a lead surrogate?
Definition: utf16.h:84
U16_GET_SUPPLEMENTARY
#define U16_GET_SUPPLEMENTARY(lead, trail)
Get a supplementary code point value (U+10000..U+10ffff) from its lead and trail surrogates.
Definition: utf16.h:112
U16_IS_SURROGATE
#define U16_IS_SURROGATE(c)
Is this code unit a surrogate (U+d800..U+dfff)?
Definition: utf16.h:75
U16_IS_LEAD
#define U16_IS_LEAD(c)
Is this code unit a lead surrogate (U+d800..U+dbff)?
Definition: utf16.h:59
U16_IS_TRAIL
#define U16_IS_TRAIL(c)
Is this code unit a trail surrogate (U+dc00..U+dfff)?
Definition: utf16.h:67
utf8.h
C API: 8-bit Unicode handling macros.
U8_COUNT_TRAIL_BYTES_UNSAFE
#define U8_COUNT_TRAIL_BYTES_UNSAFE(leadByte)
Counts the trail bytes for a UTF-8 lead byte of a valid UTF-8 sequence.
Definition: utf8.h:71
U8_IS_VALID_LEAD3_AND_T1
#define U8_IS_VALID_LEAD3_AND_T1(lead, t1)
Internal 3-byte UTF-8 validity check.
Definition: utf8.h:98
U8_IS_VALID_LEAD4_AND_T1
#define U8_IS_VALID_LEAD4_AND_T1(lead, t1)
Internal 4-byte UTF-8 validity check.
Definition: utf8.h:115
U8_IS_SINGLE
#define U8_IS_SINGLE(c)
Does this code unit (byte) encode a code point by itself (US-ASCII 0..0x7f)?
Definition: utf8.h:173
U8_LEAD3_T1_BITS
#define U8_LEAD3_T1_BITS
Internal bit vector for 3-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD3_AND_T1.
Definition: utf8.h:91
U8_LEAD4_T1_BITS
#define U8_LEAD4_T1_BITS
Internal bit vector for 4-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD4_AND_T1.
Definition: utf8.h:108
U8_IS_LEAD
#define U8_IS_LEAD(c)
Is this code unit (byte) a UTF-8 lead byte? (0xC2..0xF4)
Definition: utf8.h:181
U8_MASK_LEAD_BYTE
#define U8_MASK_LEAD_BYTE(leadByte, countTrailBytes)
Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value.
Definition: utf8.h:81
U8_IS_TRAIL
#define U8_IS_TRAIL(c)
Is this code unit (byte) a UTF-8 trail byte? (0x80..0xBF)
Definition: utf8.h:190
U_HEADER_ONLY_NAMESPACE::unsafeUTFIterator
auto unsafeUTFIterator(UnitIter iter)
UnsafeUTFIterator factory function.
Definition: utfiterator.h:2486
U_HEADER_ONLY_NAMESPACE::prv::iter_difference_t
typename std::iterator_traits< Iter >::difference_type iter_difference_t
Definition: utfiterator.h:203
U_HEADER_ONLY_NAMESPACE::prv::is_basic_string_view_v
constexpr bool is_basic_string_view_v
Definition: utfiterator.h:244
U_HEADER_ONLY_NAMESPACE::prv::forward_iterator
constexpr bool forward_iterator
Definition: utfiterator.h:207
U_HEADER_ONLY_NAMESPACE::utfIterator
auto utfIterator(UnitIter start, UnitIter p, LimitIter limit)
UTFIterator factory function for start <= p < limit.
Definition: utfiterator.h:1688
U_HEADER_ONLY_NAMESPACE::utfStringCodePoints
constexpr UTFStringCodePointsAdaptor< CP32, behavior > utfStringCodePoints
Range adaptor function object returning a UTFStringCodePoints object that represents a "range" of cod...
Definition: utfiterator.h:1926
U_HEADER_ONLY_NAMESPACE::prv::iter_value_t
typename std::iterator_traits< Iter >::value_type iter_value_t
Definition: utfiterator.h:199
U_HEADER_ONLY_NAMESPACE::prv::bidirectional_iterator
constexpr bool bidirectional_iterator
Definition: utfiterator.h:214
U_HEADER_ONLY_NAMESPACE::unsafeUTFStringCodePoints
constexpr UnsafeUTFStringCodePointsAdaptor< CP32 > unsafeUTFStringCodePoints
Range adaptor function object returning an UnsafeUTFStringCodePoints object that represents a "range"...
Definition: utfiterator.h:2658
UTFIllFormedBehavior
UTFIllFormedBehavior
Some defined behaviors for handling ill-formed Unicode strings.
Definition: utfiterator.h:149
UTF_BEHAVIOR_FFFD
@ UTF_BEHAVIOR_FFFD
Returns U+FFFD Replacement Character.
Definition: utfiterator.h:159
UTF_BEHAVIOR_SURROGATE
@ UTF_BEHAVIOR_SURROGATE
UTF-8: Not allowed; UTF-16: returns the unpaired surrogate; UTF-32: returns the surrogate code point,...
Definition: utfiterator.h:167
UTF_BEHAVIOR_NEGATIVE
@ UTF_BEHAVIOR_NEGATIVE
Returns a negative value (-1=U_SENTINEL) instead of a code point.
Definition: utfiterator.h:157
U_HEADER_ONLY_NAMESPACE::prv::range
constexpr bool range
Definition: utfiterator.h:232
utypes.h
Basic definitions for ICU, for both C and C++ APIs.
uversion.h
C API: API for accessing ICU version numbers.