/src/mongo/util/text.cpp

https://github.com/matulef/mongo · C++ · 141 lines · 103 code · 20 blank · 18 comment · 26 complexity · 9bfd564147a86bfca534277b32a1db69 MD5 · raw file

  1. // text.cpp
  2. /* Copyright 2009 10gen Inc.
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. #include "pch.h"
  17. #include "mongo/util/text.h"
  18. #include "mongo/util/unittest.h"
  19. #include "mongo/util/mongoutils/str.h"
  20. namespace mongo {
  21. inline int leadingOnes(unsigned char c) {
  22. if (c < 0x80) return 0;
  23. static const char _leadingOnes[128] = {
  24. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0x80 - 0x8F
  25. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0x90 - 0x99
  26. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0xA0 - 0xA9
  27. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0xB0 - 0xB9
  28. 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // 0xC0 - 0xC9
  29. 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // 0xD0 - 0xD9
  30. 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // 0xE0 - 0xE9
  31. 4, 4, 4, 4, 4, 4, 4, 4, // 0xF0 - 0xF7
  32. 5, 5, 5, 5, // 0xF8 - 0xFB
  33. 6, 6, // 0xFC - 0xFD
  34. 7, // 0xFE
  35. 8, // 0xFF
  36. };
  37. return _leadingOnes[c & 0x7f];
  38. }
  39. bool isValidUTF8(const char *s) {
  40. int left = 0; // how many bytes are left in the current codepoint
  41. while (*s) {
  42. const unsigned char c = (unsigned char) *(s++);
  43. const int ones = leadingOnes(c);
  44. if (left) {
  45. if (ones != 1) return false; // should be a continuation byte
  46. left--;
  47. }
  48. else {
  49. if (ones == 0) continue; // ASCII byte
  50. if (ones == 1) return false; // unexpected continuation byte
  51. if (c > 0xF4) return false; // codepoint too large (< 0x10FFFF)
  52. if (c == 0xC0 || c == 0xC1) return false; // codepoints <= 0x7F shouldn't be 2 bytes
  53. // still valid
  54. left = ones-1;
  55. }
  56. }
  57. if (left!=0) return false; // string ended mid-codepoint
  58. return true;
  59. }
  60. #if defined(_WIN32)
  61. std::string toUtf8String(const std::wstring& wide) {
  62. if (wide.size() > boost::integer_traits<int>::const_max)
  63. throw std::length_error(
  64. "Wide string cannot be more than INT_MAX characters long.");
  65. if (wide.size() == 0)
  66. return "";
  67. // Calculate necessary buffer size
  68. int len = ::WideCharToMultiByte(
  69. CP_UTF8, 0, wide.c_str(), static_cast<int>(wide.size()),
  70. NULL, 0, NULL, NULL);
  71. // Perform actual conversion
  72. if (len > 0) {
  73. std::vector<char> buffer(len);
  74. len = ::WideCharToMultiByte(
  75. CP_UTF8, 0, wide.c_str(), static_cast<int>(wide.size()),
  76. &buffer[0], static_cast<int>(buffer.size()), NULL, NULL);
  77. if (len > 0) {
  78. verify(len == static_cast<int>(buffer.size()));
  79. return std::string(&buffer[0], buffer.size());
  80. }
  81. }
  82. msgasserted( 16091 ,
  83. mongoutils::str::stream() << "can't wstring to utf8: " << ::GetLastError() );
  84. return "";
  85. }
  86. #if defined(_UNICODE)
  87. std::wstring toWideString(const char *s) {
  88. std::basic_ostringstream<TCHAR> buf;
  89. buf << s;
  90. return buf.str();
  91. }
  92. #endif
  93. WindowsCommandLine::WindowsCommandLine( int argc, wchar_t* argvW[] ) {
  94. vector < string > utf8args;
  95. vector < size_t > utf8argLength;
  96. size_t blockSize = argc * sizeof( char * );
  97. size_t blockPtr = blockSize;
  98. for ( int i = 0; i < argc; ++i ) {
  99. utf8args.push_back( toUtf8String( argvW[ i ] ) );
  100. size_t argLength = utf8args[ i ].length() + 1;
  101. utf8argLength.push_back( argLength );
  102. blockSize += argLength;
  103. }
  104. _argv = reinterpret_cast< char** >( malloc( blockSize ) );
  105. for ( int i = 0; i < argc; ++i ) {
  106. _argv[ i ] = reinterpret_cast< char * >( _argv ) + blockPtr;
  107. strcpy_s( _argv[ i ], utf8argLength[ i ], utf8args[ i ].c_str() );
  108. blockPtr += utf8argLength[ i ];
  109. }
  110. }
  111. WindowsCommandLine::~WindowsCommandLine() {
  112. free( _argv );
  113. }
  114. #endif // #if defined(_WIN32)
  115. struct TextUnitTest : public UnitTest {
  116. void run() {
  117. verify( parseLL("123") == 123 );
  118. verify( parseLL("-123000000000") == -123000000000LL );
  119. }
  120. } textUnitTest;
  121. }