sousa-gecko/xpcom/tests/gtest/TestUTF.cpp

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "UTFStrings.h"
#include "gtest/gtest.h"
#include "mozilla/HashFunctions.h"
#include "mozilla/Utf16.h"
#include "nsHashKeys.h"
#include "nsReadableUtils.h"
#include "nsString.h"

using namespace mozilla;

namespace TestUTF {

TEST(UTF, Valid)
{
  for (unsigned int i = 0; i < std::size(ValidStrings); ++i) {
    nsDependentCString str8(ValidStrings[i].m8);
    nsDependentString str16(ValidStrings[i].m16);

    EXPECT_TRUE(NS_ConvertUTF16toUTF8(str16).Equals(str8));

    EXPECT_TRUE(NS_ConvertUTF8toUTF16(str8).Equals(str16));

    nsCString tmp8("string ");
    AppendUTF16toUTF8(str16, tmp8);
    EXPECT_TRUE(tmp8.Equals("string "_ns + str8));

    nsString tmp16(u"string "_ns);
    AppendUTF8toUTF16(str8, tmp16);
    EXPECT_TRUE(tmp16.Equals(u"string "_ns + str16));

    EXPECT_EQ(CompareUTF8toUTF16(str8, str16), 0);
  }
}

TEST(UTF, Invalid16)
{
  for (unsigned int i = 0; i < std::size(Invalid16Strings); ++i) {
    nsDependentString str16(Invalid16Strings[i].m16);
    nsDependentCString str8(Invalid16Strings[i].m8);

    EXPECT_TRUE(NS_ConvertUTF16toUTF8(str16).Equals(str8));

    nsCString tmp8("string ");
    AppendUTF16toUTF8(str16, tmp8);
    EXPECT_TRUE(tmp8.Equals("string "_ns + str8));

    EXPECT_EQ(CompareUTF8toUTF16(str8, str16), 0);
  }
}

TEST(UTF, Invalid8)
{
  for (unsigned int i = 0; i < std::size(Invalid8Strings); ++i) {
    nsDependentString str16(Invalid8Strings[i].m16);
    nsDependentCString str8(Invalid8Strings[i].m8);

    EXPECT_TRUE(NS_ConvertUTF8toUTF16(str8).Equals(str16));

    nsString tmp16(u"string "_ns);
    AppendUTF8toUTF16(str8, tmp16);
    EXPECT_TRUE(tmp16.Equals(u"string "_ns + str16));

    EXPECT_EQ(CompareUTF8toUTF16(str8, str16), 0);
  }
}

TEST(UTF, Malformed8)
{
  for (unsigned int i = 0; i < std::size(Malformed8Strings); ++i) {
    nsDependentString str16(Malformed8Strings[i].m16);
    nsDependentCString str8(Malformed8Strings[i].m8);

    EXPECT_TRUE(NS_ConvertUTF8toUTF16(str8).Equals(str16));

    nsString tmp16(u"string "_ns);
    AppendUTF8toUTF16(str8, tmp16);
    EXPECT_TRUE(tmp16.Equals(u"string "_ns + str16));

    EXPECT_EQ(CompareUTF8toUTF16(str8, str16), 0);
  }
}

static HashNumber HashStringUsingUTF16Hasher(const nsAString& aString) {
  mozilla::detail::UTF16Hasher hasher;
  for (char16_t ch : Span<const char16_t>(aString)) {
    hasher.Add(ch);
  }
  return hasher.Finish();
}

TEST(UTF, Hash16)
{
  for (unsigned int i = 0; i < std::size(ValidStrings); ++i) {
    nsDependentCString str8(ValidStrings[i].m8);
    nsDependentString str16(ValidStrings[i].m16);
    EXPECT_EQ(HashString(str16), HashUTF8AsUTF16(str8.get(), str8.Length()));
    EXPECT_EQ(HashString(str16), HashStringUsingUTF16Hasher(str16));
  }

  for (unsigned int i = 0; i < std::size(Invalid8Strings); ++i) {
    nsDependentCString str8(Invalid8Strings[i].m8);
    nsDependentString str16(Invalid8Strings[i].m16);
    EXPECT_EQ(HashString(str16), HashUTF8AsUTF16(str8.get(), str8.Length()));
    EXPECT_EQ(HashString(str16), HashStringUsingUTF16Hasher(str16));
  }

  for (unsigned int i = 0; i < std::size(Malformed8Strings); ++i) {
    nsDependentCString str8(Malformed8Strings[i].m8);
    nsDependentString str16(Malformed8Strings[i].m16);
    EXPECT_EQ(HashString(str16), HashUTF8AsUTF16(str8.get(), str8.Length()));
    EXPECT_EQ(HashString(str16), HashStringUsingUTF16Hasher(str16));
  }
}

/**
 * This tests the handling of a non-ascii character at various locations in a
 * UTF-16 string that is being converted to UTF-8.
 */
static void NonASCII16_helper(const size_t aStrSize) {
  const size_t kTestSize = aStrSize;
  const size_t kMaxASCII = 0x80;
  const char16_t kUTF16Char = 0xC9;
  const char kUTF8Surrogates[] = {char(0xC3), char(0x89)};

  // Generate a string containing only ASCII characters.
  nsString asciiString;
  asciiString.SetLength(kTestSize);
  nsCString asciiCString;
  asciiCString.SetLength(kTestSize);

  auto str_buff = asciiString.BeginWriting();
  auto cstr_buff = asciiCString.BeginWriting();
  for (size_t i = 0; i < kTestSize; i++) {
    str_buff[i] = i % kMaxASCII;
    cstr_buff[i] = i % kMaxASCII;
  }

  // Now go through and test conversion when exactly one character will
  // result in a multibyte sequence.
  for (size_t i = 0; i < kTestSize; i++) {
    // Setup the UTF-16 string.
    nsString unicodeString(asciiString);
    auto buff = unicodeString.BeginWriting();
    buff[i] = kUTF16Char;

    // Do the conversion, make sure the length increased by 1.
    nsCString dest;
    AppendUTF16toUTF8(unicodeString, dest);
    EXPECT_EQ(dest.Length(), unicodeString.Length() + 1);

    // Build up the expected UTF-8 string.
    nsCString expected;

    // First add the leading ASCII chars.
    expected.Append(asciiCString.BeginReading(), i);

    // Now append the UTF-8 pair we expect the UTF-16 unicode char to
    // be converted to.
    for (auto& c : kUTF8Surrogates) {
      expected.Append(c);
    }

    // And finish with the trailing ASCII chars.
    expected.Append(asciiCString.BeginReading() + i + 1, kTestSize - i - 1);

    EXPECT_STREQ(dest.get(), expected.get());
  }
}

TEST(UTF, NonASCII16)
{
  // Test with various string sizes to catch any special casing.
  NonASCII16_helper(1);
  NonASCII16_helper(8);
  NonASCII16_helper(16);
  NonASCII16_helper(32);
  NonASCII16_helper(512);
}

TEST(UTF, DecodeOneUtf16CodePoint)
{
  const char16_t* p = u"\u0061\U0001F4A9";
  const char16_t* end = p + 3;
  EXPECT_EQ(DecodeOneUtf16CodePoint(&p, end), 0x0061U);
  EXPECT_EQ(DecodeOneUtf16CodePoint(&p, end), 0x1F4A9U);
  EXPECT_EQ(p, end);
  const char16_t loneHigh = 0xD83D;
  p = &loneHigh;
  end = p + 1;
  EXPECT_EQ(DecodeOneUtf16CodePoint(&p, end), 0xFFFDU);
  EXPECT_EQ(p, end);
  const char16_t loneLow = 0xDCA9;
  p = &loneLow;
  end = p + 1;
  EXPECT_EQ(DecodeOneUtf16CodePoint(&p, end), 0xFFFDU);
  EXPECT_EQ(p, end);
  const char16_t loneHighStr[] = {0xD83D, 0x0061};
  p = loneHighStr;
  end = p + 2;
  EXPECT_EQ(DecodeOneUtf16CodePoint(&p, end), 0xFFFDU);
  EXPECT_EQ(DecodeOneUtf16CodePoint(&p, end), 0x0061U);
  EXPECT_EQ(p, end);
}

}  // namespace TestUTF