ab854ec035
* Compile as static library, update AMBuildScript and link to core * Update VS project files to include the library * Add UTF-8 Rewind library (v1.5.1) to third_party directory * Update ACKNOWLEDGEMENTS.txt * Move AMXX buffer in its own function * Move constants from string.inc to string_const.inc and update project files * Move stocks from string.inc to string_stocks.inc and update project files * Improve UTF-8 support in containi() and update documentation * Improve UTF-8 support in strcmp() and update documentation * Improve UTF-8 support in strfind() and update documentation Worth to be noted that this native with ignorecase set was not working properly. So broken that no one reported the issue. This adds also a safety check for "pos" parameter to not go < 0. * Improve UTF-8 support in strncmp() and update documentation * Improve UTF-8 support in equali() and update documentation * Add an option to some UTF-8 Rewind functions for avoiding invalid data to be replaced By default it replaces any invalid byte or sequence of bytes by 0xFFFD (3 bytes). It can be problematic when the input buffer is not changed (from a plugin) and that some natives need to calculate a position from the converted string. With such replacement, the position is displaced due the final string length being larger. This compiles the library as C++, because I added some silly param with a default default value which is not supported by C. * Improve UTF-8 support in replace_string/ex() and update documentation * Add is_string_category() and update documentation * Update a little testsuite plugin (and fix linux compilation) * Add mb_strotolower/upper() and update documentation * Add mb_ucfirst() and update documentation * Add mb_strtotile() and update documentation * Improve UTF-8 support in get_players() and find_player() with name/case insenstive flags set * Fix KliPPy's complain
84 lines
3.0 KiB
C
84 lines
3.0 KiB
C
/*
|
|
Copyright (C) 2014-2016 Quinten Lansu
|
|
|
|
Permission is hereby granted, free of charge, to any person
|
|
obtaining a copy of this software and associated documentation
|
|
files (the "Software"), to deal in the Software without
|
|
restriction, including without limitation the rights to use,
|
|
copy, modify, merge, publish, distribute, sublicense, and/or
|
|
sell copies of the Software, and to permit persons to whom the
|
|
Software is furnished to do so, subject to the following
|
|
conditions:
|
|
|
|
The above copyright notice and this permission notice shall be
|
|
included in all copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
|
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
OTHER DEALINGS IN THE SOFTWARE.
|
|
*/
|
|
|
|
#ifndef _UTF8REWIND_INTERNAL_STREAMING_H_
|
|
#define _UTF8REWIND_INTERNAL_STREAMING_H_
|
|
|
|
/*!
|
|
\file
|
|
\brief Streaming interface.
|
|
|
|
\cond INTERNAL
|
|
*/
|
|
|
|
#include "utf8rewind.h"
|
|
|
|
/*
|
|
UAX15-D4. Stream-Safe Text Process
|
|
|
|
This is the process of producing a Unicode string in Stream-Safe Text Format by processing that string
|
|
from start to finish, inserting U+034F COMBINING GRAPHEME JOINER (CGJ) within long sequences of
|
|
non-starters. The exact position of the inserted CGJs are determined according to the following algorithm,
|
|
which describes the generation of an output string from an input string:
|
|
|
|
* If the input string is empty, return an empty output string.
|
|
* Set nonStarterCount to zero.
|
|
* For each code point C in the input string:
|
|
* Produce the NFKD decomposition S.
|
|
* If nonStarterCount plus the number of initial non-starters in S is greater than 30, append a CGJ to
|
|
the output string and set the nonStarterCount to zero.
|
|
* Append C to the output string.
|
|
* If there are no starters in S, increment nonStarterCount by the number of code points in S; otherwise,
|
|
set nonStarterCount to the number of trailing non-starters in S (which may be zero).
|
|
* Return the output string.
|
|
*/
|
|
|
|
#define STREAM_SAFE_MAX 30
|
|
#define STREAM_BUFFER_MAX 32
|
|
|
|
typedef struct {
|
|
const char* src;
|
|
size_t src_size;
|
|
uint8_t index;
|
|
uint8_t current;
|
|
uint8_t filled;
|
|
uint8_t stable;
|
|
uint8_t last_length;
|
|
unicode_t codepoint[STREAM_BUFFER_MAX];
|
|
uint8_t quick_check[STREAM_BUFFER_MAX];
|
|
uint8_t canonical_combining_class[STREAM_BUFFER_MAX];
|
|
} StreamState;
|
|
|
|
uint8_t stream_initialize(StreamState* state, const char* input, size_t inputSize);
|
|
|
|
uint8_t stream_read(StreamState* state, const size_t* propertyIndex, const uint8_t* propertyData);
|
|
|
|
uint8_t stream_write(StreamState* state, char** output, size_t* outputSize, uint8_t* bytesWritten);
|
|
|
|
uint8_t stream_reorder(StreamState* state);
|
|
|
|
/*! \endcond */
|
|
|
|
#endif /* _UTF8REWIND_INTERNAL_STREAMING_H_ */ |