/** ****************************************************************************** * Xenia : Xbox 360 Emulator Research Project * ****************************************************************************** * Copyright 2020 Ben Vanik. All rights reserved. * * Released under the BSD license - see LICENSE in the root for more details. * ****************************************************************************** */ #include "xenia/base/utf8.h" #include #include #include #define UTF_CPP_CPLUSPLUS 201703L #include "third_party/utfcpp/source/utf8.h" namespace utfcpp = utf8; using citer = std::string_view::const_iterator; using utf8_citer = utfcpp::iterator; namespace xe::utf8 { std::string to_string(char32_t c) { std::string result; utfcpp::append(c, result); return result; } uint32_t lower_ascii(const uint32_t c) { return c >= 'A' && c <= 'Z' ? c + 32 : c; } uint32_t upper_ascii(const uint32_t c) { return c >= 'A' && c <= 'Z' ? c + 32 : c; } bool equal_ascii_case(const uint32_t l, const uint32_t r) { return l == r || lower_ascii(l) == lower_ascii(r); } std::pair make_citer(const std::string_view view) { return {utf8_citer(view.cbegin(), view.cbegin(), view.cend()), utf8_citer(view.cend(), view.cbegin(), view.cend())}; } std::pair make_citer(const utf8_citer begin, const utf8_citer end) { return {utf8_citer(begin.base(), begin.base(), end.base()), utf8_citer(end.base(), begin.base(), end.base())}; } size_t byte_length(utf8_citer begin, utf8_citer end) { return size_t(std::distance(begin.base(), end.base())); } size_t count(const std::string_view view) { return size_t(utfcpp::distance(view.cbegin(), view.cend())); } std::string lower_ascii(const std::string_view view) { auto [begin, end] = make_citer(view); std::string result; for (auto it = begin; it != end; ++it) { utfcpp::append(char32_t(lower_ascii(*it)), result); } return result; } std::string upper_ascii(const std::string_view view) { auto [begin, end] = make_citer(view); std::string result; for (auto it = begin; it != end; ++it) { utfcpp::append(char32_t(upper_ascii(*it)), result); } return result; } template inline size_t hash_fnv1a(const std::string_view view) { const size_t offset_basis = 0xCBF29CE484222325ull; const size_t prime = 0x00000100000001B3ull; auto work = [&prime](size_t hash, uint8_t byte_of_data) { hash ^= byte_of_data; hash *= prime; return hash; }; auto hash = offset_basis; auto [begin, end] = make_citer(view); for (auto it = begin; it != end; ++it) { uint32_t c; if constexpr (LOWER) { c = lower_ascii(*it); } else { c = *it; } hash = work(hash, uint8_t((c >> 0) & 0xFF)); hash = work(hash, uint8_t((c >> 8) & 0xFF)); hash = work(hash, uint8_t((c >> 16) & 0xFF)); hash = work(hash, uint8_t((c >> 24) & 0xFF)); } return hash; } size_t hash_fnv1a(const std::string_view view) { return hash_fnv1a(view); } size_t hash_fnv1a_case(const std::string_view view) { return hash_fnv1a(view); } // TODO(gibbed): this is a separate inline function instead of inline within // split due to a Clang bug: reference to local binding 'needle_begin' declared // in enclosing function 'split'. inline utf8_citer find_needle(utf8_citer haystack_it, utf8_citer haystack_end, utf8_citer needle_begin, utf8_citer needle_end) { return std::find_if(haystack_it, haystack_end, [&](const auto& c) { for (auto needle = needle_begin; needle != needle_end; ++needle) { if (c == *needle) { return true; } } return false; }); } inline utf8_citer find_needle_case(utf8_citer haystack_it, utf8_citer haystack_end, utf8_citer needle_begin, utf8_citer needle_end) { return std::find_if(haystack_it, haystack_end, [&](const auto& c) { for (auto needle = needle_begin; needle != needle_end; ++needle) { if (equal_ascii_case(c, *needle)) { return true; } } return false; }); } std::vector split(const std::string_view haystack, const std::string_view needles, bool remove_empty) { std::vector result; auto [haystack_begin, haystack_end] = make_citer(haystack); auto [needle_begin, needle_end] = make_citer(needles); auto it = haystack_begin; auto last = it; for (;;) { it = find_needle(it, haystack_end, needle_begin, needle_end); if (it == haystack_end) { break; } if (it != last) { auto offset = byte_length(haystack_begin, last); auto length = byte_length(haystack_begin, it) - offset; result.push_back(haystack.substr(offset, length)); } else if (!remove_empty) { result.push_back(""); } ++it; last = it; } if (last != haystack_end) { auto offset = byte_length(haystack_begin, last); result.push_back(haystack.substr(offset)); } return result; } bool equal_z(const std::string_view left, const std::string_view right) { if (!left.size()) { return !right.size(); } else if (!right.size()) { return false; } auto [left_begin, left_end] = make_citer(left); auto [right_begin, right_end] = make_citer(right); auto left_it = left_begin; auto right_it = right_begin; for (; left_it != left_end && *left_it != 0 && right_it != right_end && *right_it != 0; ++left_it, ++right_it) { if (*left_it != *right_it) { return false; } } return (left_it == left_end || *left_it == 0) && (right_it == right_end || *right_it == 0); } bool equal_case(const std::string_view left, const std::string_view right) { if (!left.size()) { return !right.size(); } else if (!right.size()) { return false; } auto [left_begin, left_end] = make_citer(left); auto [right_begin, right_end] = make_citer(right); return std::equal(left_begin, left_end, right_begin, right_end, equal_ascii_case); } bool equal_case_z(const std::string_view left, const std::string_view right) { if (!left.size()) { return !right.size(); } else if (!right.size()) { return false; } auto [left_begin, left_end] = make_citer(left); auto [right_begin, right_end] = make_citer(right); auto left_it = left_begin; auto right_it = right_begin; for (; left_it != left_end && *left_it != 0 && right_it != right_end && *right_it != 0; ++left_it, ++right_it) { if (!equal_ascii_case(*left_it, *right_it)) { return false; } } return (left_it == left_end || *left_it == 0) && (right_it == right_end || *right_it == 0); } std::string_view::size_type find_any_of(const std::string_view haystack, const std::string_view needles) { if (needles.empty()) { return std::string_view::size_type(0); } else if (haystack.empty()) { return std::string_view::npos; } auto [haystack_begin, haystack_end] = make_citer(haystack); auto [needle_begin, needle_end] = make_citer(needles); auto it = find_needle(haystack_begin, haystack_end, needle_begin, needle_end); if (it == haystack_end) { return std::string_view::npos; } return std::string_view::size_type(byte_length(haystack_begin, it)); } std::string_view::size_type find_any_of_case(const std::string_view haystack, const std::string_view needles) { if (needles.empty()) { return std::string_view::size_type(0); } else if (haystack.empty()) { return std::string_view::npos; } auto [haystack_begin, haystack_end] = make_citer(haystack); auto [needle_begin, needle_end] = make_citer(needles); auto it = find_needle_case(haystack_begin, haystack_end, needle_begin, needle_end); if (it == haystack_end) { return std::string_view::npos; } return std::string_view::size_type(byte_length(haystack_begin, it)); } std::string_view::size_type find_first_of(const std::string_view haystack, const std::string_view needle) { if (needle.empty()) { return std::string_view::size_type(0); } else if (haystack.empty()) { return std::string_view::npos; } auto [haystack_begin, haystack_end] = make_citer(haystack); auto [needle_begin, needle_end] = make_citer(needle); auto needle_count = count(needle); auto it = haystack_begin; for (; it != haystack_end; ++it) { it = std::find(it, haystack_end, *needle_begin); if (it == haystack_end) { return std::string_view::npos; } auto end = it; for (size_t i = 0; i < needle_count; ++i) { if (end == haystack_end) { // not enough room in target for search return std::string_view::npos; } ++end; } auto [sub_start, sub_end] = make_citer(it, end); if (std::equal(needle_begin, needle_end, sub_start, sub_end)) { return std::string_view::size_type(byte_length(haystack_begin, it)); } } return std::string_view::npos; } std::string_view::size_type find_first_of_case(const std::string_view haystack, const std::string_view needle) { if (needle.empty()) { return std::string_view::size_type(0); } else if (haystack.empty()) { return std::string_view::npos; } auto [haystack_begin, haystack_end] = make_citer(haystack); auto [needle_begin, needle_end] = make_citer(needle); auto needle_count = count(needle); auto nc = *needle_begin; auto it = haystack_begin; for (; it != haystack_end; ++it) { it = std::find_if(it, haystack_end, [&nc](const uint32_t& c) { return equal_ascii_case(nc, c); }); if (it == haystack_end) { return std::string_view::npos; } auto end = it; for (size_t i = 0; i < needle_count; ++i) { if (end == haystack_end) { // not enough room in target for search return std::string_view::npos; } ++end; } auto [sub_start, sub_end] = make_citer(it, end); if (std::equal(needle_begin, needle_end, sub_start, sub_end, equal_ascii_case)) { return std::string_view::size_type(byte_length(haystack_begin, it)); } } return std::string_view::npos; } bool starts_with(const std::string_view haystack, char32_t needle) { if (haystack.empty()) { return false; } auto [it, end] = make_citer(haystack); return *it == uint32_t(needle); } bool starts_with(const std::string_view haystack, const std::string_view needle) { if (needle.empty()) { return true; } else if (haystack.empty()) { return false; } auto [haystack_begin, haystack_end] = make_citer(haystack); auto [needle_begin, needle_end] = make_citer(needle); auto needle_count = count(needle); auto it = haystack_begin; auto end = it; for (size_t i = 0; i < needle_count; ++i) { if (end == haystack_end) { // not enough room in target for search return false; } ++end; } auto [sub_start, sub_end] = make_citer(it, end); return std::equal(needle_begin, needle_end, sub_start, sub_end); } bool starts_with_case(const std::string_view haystack, const std::string_view needle) { if (needle.empty()) { return true; } else if (haystack.empty()) { return false; } auto [haystack_begin, haystack_end] = make_citer(haystack); auto [needle_begin, needle_end] = make_citer(needle); auto needle_count = count(needle); auto it = haystack_begin; auto end = it; for (size_t i = 0; i < needle_count; ++i) { if (end == haystack_end) { // not enough room in target for search return false; } ++end; } auto [sub_start, sub_end] = make_citer(it, end); return std::equal(needle_begin, needle_end, sub_start, sub_end, equal_ascii_case); } bool ends_with(const std::string_view haystack, const std::string_view needle) { if (needle.empty()) { return true; } else if (haystack.empty()) { return false; } auto [haystack_begin, haystack_end] = make_citer(haystack); auto [needle_begin, needle_end] = make_citer(needle); auto needle_count = count(needle); auto it = haystack_end; auto end = it; --it; for (size_t i = 1; i < needle_count; ++i) { if (it == haystack_begin) { // not enough room in target for search return false; } --it; } auto [sub_start, sub_end] = make_citer(it, end); return std::equal(needle_begin, needle_end, sub_start, sub_end); } bool ends_with_case(const std::string_view haystack, const std::string_view needle) { if (needle.empty()) { return true; } else if (haystack.empty()) { return false; } auto [haystack_begin, haystack_end] = make_citer(haystack); auto [needle_begin, needle_end] = make_citer(needle); auto needle_count = count(needle); auto it = haystack_end; auto end = it; --it; for (size_t i = 0; i < needle_count; ++i) { if (it == haystack_begin) { // not enough room in target for search return false; } --it; } auto [sub_start, sub_end] = make_citer(it, end); return std::equal(needle_begin, needle_end, sub_start, sub_end, equal_ascii_case); } std::vector split_path(const std::string_view path) { return split(path, u8"\\/", true); } std::string join_paths(const std::string_view left_path, const std::string_view right_path, char32_t separator) { if (!left_path.length()) { return std::string(right_path); } else if (!right_path.length()) { return std::string(left_path); } utf8_citer it; std::tie(std::ignore, it) = make_citer(left_path); --it; std::string result = std::string(left_path); if (*it != static_cast(separator)) { utfcpp::append(separator, result); } return result + std::string(right_path); } std::string join_paths(const std::vector& paths, char32_t separator) { std::string result; auto it = paths.cbegin(); if (it != paths.cend()) { result = *it++; for (; it != paths.cend(); ++it) { result = join_paths(result, *it, separator); } } return result; } std::string join_paths(const std::vector& paths, char32_t separator) { std::string result; auto it = paths.cbegin(); if (it != paths.cend()) { result = *it++; for (; it != paths.cend(); ++it) { result = join_paths(result, *it, separator); } } return result; } std::string fix_path_separators(const std::string_view path, char32_t new_separator) { if (path.empty()) { return std::string(); } // Swap all separators to new_sep. const char32_t old_separator = new_separator == U'\\' ? U'/' : U'\\'; auto [path_begin, path_end] = make_citer(path); std::string result; auto it = path_begin; auto last = it; auto is_separator = [old_separator, new_separator](char32_t c) { return c == uint32_t(old_separator) || c == uint32_t(new_separator); }; // Begins with a separator if (is_separator(*it)) { utfcpp::append(new_separator, result); ++it; last = it; } for (;;) { it = std::find_if(it, path_end, is_separator); if (it == path_end) { break; } if (it != last) { auto offset = byte_length(path_begin, last); auto length = byte_length(path_begin, it) - offset; result += path.substr(offset, length); utfcpp::append(new_separator, result); } ++it; last = it; } if (last == path_begin) { return std::string(path); } if (last != path_end) { auto offset = byte_length(path_begin, last); result += path.substr(offset); } return result; } std::string find_name_from_path(const std::string_view path, char32_t separator) { if (path.empty()) { return std::string(); } auto [begin, end] = make_citer(path); auto it = end; --it; // skip trailing separators at the end of the path while (*it == uint32_t(separator)) { if (it == begin) { // path is all separators, name is empty return std::string(); } --it; } // update end so it is before any trailing separators end = std::next(it); // skip non-separators while (*it != uint32_t(separator)) { if (it == begin) { break; } --it; } // if the iterator is on a separator, advance if (*it == uint32_t(separator)) { ++it; } auto offset = byte_length(begin, it); auto length = byte_length(it, end); return std::string(path.substr(offset, length)); } std::string find_base_name_from_path(const std::string_view path, char32_t separator) { auto name = find_name_from_path(path, separator); if (!name.length()) { return std::string(); } auto [begin, end] = make_citer(name); auto it = end; --it; while (it != begin) { if (*it == uint32_t('.')) { break; } --it; } if (it == begin) { return name; } auto length = byte_length(it, end); auto offset = name.length() - length; return std::string(name.substr(0, offset)); } std::string find_base_path(const std::string_view path, char32_t separator) { if (path.empty()) { return std::string(); } auto [begin, end] = make_citer(path); auto it = end; --it; // skip trailing separators at the end of the path while (*it == uint32_t(separator)) { if (it == begin) { return std::string(); } --it; } // skip non-separators while (*it != uint32_t(separator)) { if (it == begin) { // there are no separators, base path is empty return std::string(); } --it; } // skip trailing separators at the end of the base path while (*it == uint32_t(separator)) { if (it == begin) { // base path is all separators, base path is empty return std::string(); } --it; } auto length = byte_length(begin, std::next(it)); return std::string(path.substr(0, length)); } std::string canonicalize_path(const std::string_view path, char32_t separator) { if (path.empty()) { return std::string(); } auto is_rooted = starts_with(path, separator); auto parts = split_path(path); for (auto it = parts.begin(); it != parts.end();) { const auto& part = *it; if (part == ".") { // Potential marker for current directory. it = parts.erase(it); } else if (part == "..") { // Ensure we don't override the device name. if (it != parts.begin()) { auto prev = std::prev(it); if (!ends_with(*prev, ":")) { it = parts.erase(prev); } } it = parts.erase(it); } else { ++it; } } return !is_rooted ? join_paths(parts, separator) : to_string(separator) + join_paths(parts, separator); } } // namespace xe::utf8