/** \file * Implementation of the ANTLR3 string and string factory classes */ // [The "BSD licence"] // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC // http://www.temporal-wave.com // http://www.linkedin.com/in/jimidle // // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // 3. The name of the author may not be used to endorse or promote products // derived from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include /* Factory API */ static pANTLR3_STRING newRaw8 (pANTLR3_STRING_FACTORY factory); static pANTLR3_STRING newRawUTF16 (pANTLR3_STRING_FACTORY factory); static pANTLR3_STRING newSize8 (pANTLR3_STRING_FACTORY factory, ANTLR3_UINT32 size); static pANTLR3_STRING newSizeUTF16 (pANTLR3_STRING_FACTORY factory, ANTLR3_UINT32 size); static pANTLR3_STRING newPtr8 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string, ANTLR3_UINT32 size); static pANTLR3_STRING newPtrUTF16_8 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string, ANTLR3_UINT32 size); static pANTLR3_STRING newPtrUTF16_UTF16 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string, ANTLR3_UINT32 size); static pANTLR3_STRING newStr8 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string); static pANTLR3_STRING newStrUTF16_8 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string); static pANTLR3_STRING newStrUTF16_UTF16 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string); static void destroy (pANTLR3_STRING_FACTORY factory, pANTLR3_STRING string); static pANTLR3_STRING printable8 (pANTLR3_STRING_FACTORY factory, pANTLR3_STRING string); static pANTLR3_STRING printableUTF16 (pANTLR3_STRING_FACTORY factory, pANTLR3_STRING string); static void closeFactory(pANTLR3_STRING_FACTORY factory); /* String API */ static pANTLR3_UINT8 set8 (pANTLR3_STRING string, const char * chars); static pANTLR3_UINT8 setUTF16_8 (pANTLR3_STRING string, const char * chars); static pANTLR3_UINT8 setUTF16_UTF16 (pANTLR3_STRING string, const char * chars); static pANTLR3_UINT8 append8 (pANTLR3_STRING string, const char * newbit); static pANTLR3_UINT8 appendUTF16_8 (pANTLR3_STRING string, const char * newbit); static pANTLR3_UINT8 appendUTF16_UTF16 (pANTLR3_STRING string, const char * newbit); static pANTLR3_UINT8 insert8 (pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit); static pANTLR3_UINT8 insertUTF16_8 (pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit); static pANTLR3_UINT8 insertUTF16_UTF16 (pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit); static pANTLR3_UINT8 setS (pANTLR3_STRING string, pANTLR3_STRING chars); static pANTLR3_UINT8 appendS (pANTLR3_STRING string, pANTLR3_STRING newbit); static pANTLR3_UINT8 insertS (pANTLR3_STRING string, ANTLR3_UINT32 point, pANTLR3_STRING newbit); static pANTLR3_UINT8 addc8 (pANTLR3_STRING string, ANTLR3_UINT32 c); static pANTLR3_UINT8 addcUTF16 (pANTLR3_STRING string, ANTLR3_UINT32 c); static pANTLR3_UINT8 addi8 (pANTLR3_STRING string, ANTLR3_INT32 i); static pANTLR3_UINT8 addiUTF16 (pANTLR3_STRING string, ANTLR3_INT32 i); static pANTLR3_UINT8 inserti8 (pANTLR3_STRING string, ANTLR3_UINT32 point, ANTLR3_INT32 i); static pANTLR3_UINT8 insertiUTF16 (pANTLR3_STRING string, ANTLR3_UINT32 point, ANTLR3_INT32 i); static ANTLR3_UINT32 compare8 (pANTLR3_STRING string, const char * compStr); static ANTLR3_UINT32 compareUTF16_8 (pANTLR3_STRING string, const char * compStr); static ANTLR3_UINT32 compareUTF16_UTF16(pANTLR3_STRING string, const char * compStr); static ANTLR3_UINT32 compareS (pANTLR3_STRING string, pANTLR3_STRING compStr); static ANTLR3_UCHAR charAt8 (pANTLR3_STRING string, ANTLR3_UINT32 offset); static ANTLR3_UCHAR charAtUTF16 (pANTLR3_STRING string, ANTLR3_UINT32 offset); static pANTLR3_STRING subString8 (pANTLR3_STRING string, ANTLR3_UINT32 startIndex, ANTLR3_UINT32 endIndex); static pANTLR3_STRING subStringUTF16 (pANTLR3_STRING string, ANTLR3_UINT32 startIndex, ANTLR3_UINT32 endIndex); static ANTLR3_INT32 toInt32_8 (pANTLR3_STRING string); static ANTLR3_INT32 toInt32_UTF16 (pANTLR3_STRING string); static pANTLR3_STRING to8_8 (pANTLR3_STRING string); static pANTLR3_STRING to8_UTF16 (pANTLR3_STRING string); static pANTLR3_STRING toUTF8_8 (pANTLR3_STRING string); static pANTLR3_STRING toUTF8_UTF16 (pANTLR3_STRING string); /* Local helpers */ static void stringInit8 (pANTLR3_STRING string); static void stringInitUTF16 (pANTLR3_STRING string); static void ANTLR3_CDECL stringFree (pANTLR3_STRING string); ANTLR3_API pANTLR3_STRING_FACTORY antlr3StringFactoryNew(ANTLR3_UINT32 encoding) { pANTLR3_STRING_FACTORY factory; /* Allocate memory */ factory = (pANTLR3_STRING_FACTORY) ANTLR3_CALLOC(1, sizeof(ANTLR3_STRING_FACTORY)); if (factory == NULL) { return NULL; } /* Now we make a new list to track the strings. */ factory->strings = antlr3VectorNew(0); factory->index = 0; if (factory->strings == NULL) { ANTLR3_FREE(factory); return NULL; } // Install the API // // TODO: These encodings need equivalent functions to // UTF16 and 8Bit if I am going to support those encodings in the STRING stuff. // The STRING stuff was intended as a quick and dirty hack for people that did not // want to worry about memory and performance very much, but nobody ever reads the // notes or comments or uses the email list search. I want to discourage using these // interfaces as it is much more efficient to use the pointers within the tokens // directly, so I am not implementing the string stuff for the newer encodings. // We install the standard 8 and 16 bit functions for the UTF 8 and 16 but they // will not be useful beyond returning the text. // switch(encoding) { case ANTLR3_ENC_UTF32: break; case ANTLR3_ENC_UTF32BE: break; case ANTLR3_ENC_UTF32LE: break; case ANTLR3_ENC_UTF16BE: case ANTLR3_ENC_UTF16LE: case ANTLR3_ENC_UTF16: factory->newRaw = newRawUTF16; factory->newSize = newSizeUTF16; factory->newPtr = newPtrUTF16_UTF16; factory->newPtr8 = newPtrUTF16_8; factory->newStr = newStrUTF16_UTF16; factory->newStr8 = newStrUTF16_8; factory->printable = printableUTF16; factory->destroy = destroy; factory->close = closeFactory; break; case ANTLR3_ENC_UTF8: case ANTLR3_ENC_EBCDIC: case ANTLR3_ENC_8BIT: default: factory->newRaw = newRaw8; factory->newSize = newSize8; factory->newPtr = newPtr8; factory->newPtr8 = newPtr8; factory->newStr = newStr8; factory->newStr8 = newStr8; factory->printable = printable8; factory->destroy = destroy; factory->close = closeFactory; break; } return factory; } /** * * \param factory * \return */ static pANTLR3_STRING newRaw8 (pANTLR3_STRING_FACTORY factory) { pANTLR3_STRING string; string = (pANTLR3_STRING) ANTLR3_MALLOC(sizeof(ANTLR3_STRING)); if (string == NULL) { return NULL; } /* Structure is allocated, now fill in the API etc. */ stringInit8(string); string->factory = factory; /* Add the string into the allocated list */ factory->strings->set(factory->strings, factory->index, (void *) string, (void (ANTLR3_CDECL *)(void *))(stringFree), ANTLR3_TRUE); string->index = factory->index++; return string; } /** * * \param factory * \return */ static pANTLR3_STRING newRawUTF16 (pANTLR3_STRING_FACTORY factory) { pANTLR3_STRING string; string = (pANTLR3_STRING) ANTLR3_MALLOC(sizeof(ANTLR3_STRING)); if (string == NULL) { return NULL; } /* Structure is allocated, now fill in the API etc. */ stringInitUTF16(string); string->factory = factory; /* Add the string into the allocated list */ factory->strings->set(factory->strings, factory->index, (void *) string, (void (ANTLR3_CDECL *)(void *))(stringFree), ANTLR3_TRUE); string->index = factory->index++; return string; } static void ANTLR3_CDECL stringFree (pANTLR3_STRING string) { /* First free the string itself if there was anything in it */ if (string->chars) { ANTLR3_FREE(string->chars); } /* Now free the space for this string */ ANTLR3_FREE(string); return; } /** * * \param string * \return */ static void stringInit8 (pANTLR3_STRING string) { string->len = 0; string->size = 0; string->chars = NULL; string->encoding = ANTLR3_ENC_8BIT ; /* API for 8 bit strings*/ string->set = set8; string->set8 = set8; string->append = append8; string->append8 = append8; string->insert = insert8; string->insert8 = insert8; string->addi = addi8; string->inserti = inserti8; string->addc = addc8; string->charAt = charAt8; string->compare = compare8; string->compare8 = compare8; string->subString = subString8; string->toInt32 = toInt32_8; string->to8 = to8_8; string->toUTF8 = toUTF8_8; string->compareS = compareS; string->setS = setS; string->appendS = appendS; string->insertS = insertS; } /** * * \param string * \return */ static void stringInitUTF16 (pANTLR3_STRING string) { string->len = 0; string->size = 0; string->chars = NULL; string->encoding = ANTLR3_ENC_8BIT; /* API for UTF16 strings */ string->set = setUTF16_UTF16; string->set8 = setUTF16_8; string->append = appendUTF16_UTF16; string->append8 = appendUTF16_8; string->insert = insertUTF16_UTF16; string->insert8 = insertUTF16_8; string->addi = addiUTF16; string->inserti = insertiUTF16; string->addc = addcUTF16; string->charAt = charAtUTF16; string->compare = compareUTF16_UTF16; string->compare8 = compareUTF16_8; string->subString = subStringUTF16; string->toInt32 = toInt32_UTF16; string->to8 = to8_UTF16; string->toUTF8 = toUTF8_UTF16; string->compareS = compareS; string->setS = setS; string->appendS = appendS; string->insertS = insertS; } /** * * \param string * \return * TODO: Implement UTF-8 */ static void stringInitUTF8 (pANTLR3_STRING string) { string->len = 0; string->size = 0; string->chars = NULL; /* API */ } // Convert an 8 bit string into a UTF8 representation, which is in fact just the string itself // a memcpy as we make no assumptions about the 8 bit encoding. // static pANTLR3_STRING toUTF8_8 (pANTLR3_STRING string) { return string->factory->newPtr(string->factory, (pANTLR3_UINT8)(string->chars), string->len); } // Convert a UTF16 string into a UTF8 representation using the Unicode.org // supplied C algorithms, which are now contained within the ANTLR3 C runtime // as permitted by the Unicode license (within the source code antlr3convertutf.c/.h // UCS2 has the same encoding as UTF16 so we can use UTF16 converter. // static pANTLR3_STRING toUTF8_UTF16 (pANTLR3_STRING string) { UTF8 * outputEnd; UTF16 * inputEnd; pANTLR3_STRING utf8String; ConversionResult cResult; // Allocate the output buffer, which needs to accommodate potentially // 3X (in bytes) the input size (in chars). // utf8String = string->factory->newStr8(string->factory, (pANTLR3_UINT8)""); if (utf8String != NULL) { // Free existing allocation // ANTLR3_FREE(utf8String->chars); // Reallocate according to maximum expected size // utf8String->size = string->len *3; utf8String->chars = (pANTLR3_UINT8)ANTLR3_MALLOC(utf8String->size +1); if (utf8String->chars != NULL) { inputEnd = (UTF16 *) (string->chars); outputEnd = (UTF8 *) (utf8String->chars); // Call the Unicode converter // cResult = ConvertUTF16toUTF8 ( (const UTF16**)&inputEnd, ((const UTF16 *)(string->chars)) + string->len, &outputEnd, outputEnd + utf8String->size - 1, lenientConversion ); // We don't really care if things failed or not here, we just converted // everything that was vaguely possible and stopped when it wasn't. It is // up to the grammar programmer to verify that the input is sensible. // utf8String->len = ANTLR3_UINT32_CAST(((pANTLR3_UINT8)outputEnd) - utf8String->chars); *(outputEnd+1) = '\0'; // Always null terminate } } return utf8String; } /** * Creates a new string with enough capacity for size 8 bit characters plus a terminator. * * \param[in] factory - Pointer to the string factory that owns strings * \param[in] size - In characters * \return pointer to the new string. */ static pANTLR3_STRING newSize8 (pANTLR3_STRING_FACTORY factory, ANTLR3_UINT32 size) { pANTLR3_STRING string; string = factory->newRaw(factory); if (string == NULL) { return string; } /* Always add one more byte for a terminator ;-) */ string->chars = (pANTLR3_UINT8) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_UINT8) * (size+1))); if (string->chars != NULL) { *(string->chars) = '\0'; string->size = size + 1; } return string; } /** * Creates a new string with enough capacity for size UTF16 characters plus a terminator. * * \param[in] factory - Pointer to the string factory that owns strings * \param[in] size - In characters (count double for surrogate pairs!!!) * \return pointer to the new string. */ static pANTLR3_STRING newSizeUTF16 (pANTLR3_STRING_FACTORY factory, ANTLR3_UINT32 size) { pANTLR3_STRING string; string = factory->newRaw(factory); if (string == NULL) { return string; } /* Always add one more byte for a terminator ;-) */ string->chars = (pANTLR3_UINT8) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_UINT16) * (size+1))); if (string->chars != NULL) { *(string->chars) = '\0'; string->size = size+1; /* Size is always in characters, as is len */ } return string; } /** Creates a new 8 bit string initialized with the 8 bit characters at the * supplied ptr, of pre-determined size. * \param[in] factory - Pointer to the string factory that owns the strings * \param[in] ptr - Pointer to 8 bit encoded characters * \return pointer to the new string */ static pANTLR3_STRING newPtr8 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr, ANTLR3_UINT32 size) { pANTLR3_STRING string; string = factory->newSize(factory, size); if (string == NULL) { return NULL; } if (size <= 0) { return string; } if (ptr != NULL) { ANTLR3_MEMMOVE(string->chars, (const void *)ptr, size); *(string->chars + size) = '\0'; /* Terminate, these strings are usually used for Token streams and printing etc. */ string->len = size; } return string; } /** Creates a new UTF16 string initialized with the 8 bit characters at the * supplied 8 bit character ptr, of pre-determined size. * \param[in] factory - Pointer to the string factory that owns the strings * \param[in] ptr - Pointer to 8 bit encoded characters * \return pointer to the new string */ static pANTLR3_STRING newPtrUTF16_8 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr, ANTLR3_UINT32 size) { pANTLR3_STRING string; /* newSize accepts size in characters, not bytes */ string = factory->newSize(factory, size); if (string == NULL) { return NULL; } if (size <= 0) { return string; } if (ptr != NULL) { pANTLR3_UINT16 out; ANTLR3_INT32 inSize; out = (pANTLR3_UINT16)(string->chars); inSize = size; while (inSize-- > 0) { *out++ = (ANTLR3_UINT16)(*ptr++); } /* Terminate, these strings are usually used for Token streams and printing etc. */ *(((pANTLR3_UINT16)(string->chars)) + size) = '\0'; string->len = size; } return string; } /** Creates a new UTF16 string initialized with the UTF16 characters at the * supplied ptr, of pre-determined size. * \param[in] factory - Pointer to the string factory that owns the strings * \param[in] ptr - Pointer to UTF16 encoded characters * \return pointer to the new string */ static pANTLR3_STRING newPtrUTF16_UTF16 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr, ANTLR3_UINT32 size) { pANTLR3_STRING string; string = factory->newSize(factory, size); if (string == NULL) { return NULL; } if (size <= 0) { return string; } if (ptr != NULL) { ANTLR3_MEMMOVE(string->chars, (const void *)ptr, (size * sizeof(ANTLR3_UINT16))); /* Terminate, these strings are usually used for Token streams and printing etc. */ *(((pANTLR3_UINT16)(string->chars)) + size) = '\0'; string->len = size; } return string; } /** Create a new 8 bit string from the supplied, null terminated, 8 bit string pointer. * \param[in] factory - Pointer to the string factory that owns strings. * \param[in] ptr - Pointer to the 8 bit encoded string * \return Pointer to the newly initialized string */ static pANTLR3_STRING newStr8 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr) { return factory->newPtr8(factory, ptr, (ANTLR3_UINT32)strlen((const char *)ptr)); } /** Create a new UTF16 string from the supplied, null terminated, 8 bit string pointer. * \param[in] factory - Pointer to the string factory that owns strings. * \param[in] ptr - Pointer to the 8 bit encoded string * \return POinter to the newly initialized string */ static pANTLR3_STRING newStrUTF16_8 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr) { return factory->newPtr8(factory, ptr, (ANTLR3_UINT32)strlen((const char *)ptr)); } /** Create a new UTF16 string from the supplied, null terminated, UTF16 string pointer. * \param[in] factory - Pointer to the string factory that owns strings. * \param[in] ptr - Pointer to the UTF16 encoded string * \return Pointer to the newly initialized string */ static pANTLR3_STRING newStrUTF16_UTF16 (pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr) { pANTLR3_UINT16 in; ANTLR3_UINT32 count; /** First, determine the length of the input string */ in = (pANTLR3_UINT16)ptr; count = 0; while (*in++ != '\0') { count++; } return factory->newPtr(factory, ptr, count); } static void destroy (pANTLR3_STRING_FACTORY factory, pANTLR3_STRING string) { // Record which string we are deleting // ANTLR3_UINT32 strIndex = string->index; // Ensure that the string was not factory made, or we would try // to delete memory that wasn't allocated outside the factory // block. // Remove the specific indexed string from the vector // factory->strings->del(factory->strings, strIndex); // One less string in the vector, so decrement the factory index // so that the next string allocated is indexed correctly with // respect to the vector. // factory->index--; // Now we have to reindex the strings in the vector that followed // the one we just deleted. We only do this if the one we just deleted // was not the last one. // if (strIndex< factory->index) { // We must reindex the strings after the one we just deleted. // The one that follows the one we just deleted is also out // of whack, so we start there. // ANTLR3_UINT32 i; for (i = strIndex; i < factory->index; i++) { // Renumber the entry // ((pANTLR3_STRING)(factory->strings->elements[i].element))->index = i; } } // The string has been destroyed and the elements of the factory are reindexed. // } static pANTLR3_STRING printable8(pANTLR3_STRING_FACTORY factory, pANTLR3_STRING instr) { pANTLR3_STRING string; /* We don't need to be too efficient here, this is mostly for error messages and so on. */ pANTLR3_UINT8 scannedText; ANTLR3_UINT32 i; /* Assume we need as much as twice as much space to parse out the control characters */ string = factory->newSize(factory, instr->len *2 + 1); /* Scan through and replace unprintable (in terms of this routine) * characters */ scannedText = string->chars; for (i = 0; i < instr->len; i++) { if (*(instr->chars + i) == '\n') { *scannedText++ = '\\'; *scannedText++ = 'n'; } else if (*(instr->chars + i) == '\r') { *scannedText++ = '\\'; *scannedText++ = 'r'; } else if (!isprint(*(instr->chars +i))) { *scannedText++ = '?'; } else { *scannedText++ = *(instr->chars + i); } } *scannedText = '\0'; string->len = (ANTLR3_UINT32)(scannedText - string->chars); return string; } static pANTLR3_STRING printableUTF16(pANTLR3_STRING_FACTORY factory, pANTLR3_STRING instr) { pANTLR3_STRING string; /* We don't need to be too efficient here, this is mostly for error messages and so on. */ pANTLR3_UINT16 scannedText; pANTLR3_UINT16 inText; ANTLR3_UINT32 i; ANTLR3_UINT32 outLen; /* Assume we need as much as twice as much space to parse out the control characters */ string = factory->newSize(factory, instr->len *2 + 1); /* Scan through and replace unprintable (in terms of this routine) * characters */ scannedText = (pANTLR3_UINT16)(string->chars); inText = (pANTLR3_UINT16)(instr->chars); outLen = 0; for (i = 0; i < instr->len; i++) { if (*(inText + i) == '\n') { *scannedText++ = '\\'; *scannedText++ = 'n'; outLen += 2; } else if (*(inText + i) == '\r') { *scannedText++ = '\\'; *scannedText++ = 'r'; outLen += 2; } else if (!isprint(*(inText +i))) { *scannedText++ = '?'; outLen++; } else { *scannedText++ = *(inText + i); outLen++; } } *scannedText = '\0'; string->len = outLen; return string; } /** Fascist Capitalist Pig function created * to oppress the workers comrade. */ static void closeFactory (pANTLR3_STRING_FACTORY factory) { /* Delete the vector we were tracking the strings with, this will * causes all the allocated strings to be deallocated too */ factory->strings->free(factory->strings); /* Delete the space for the factory itself */ ANTLR3_FREE((void *)factory); } static pANTLR3_UINT8 append8 (pANTLR3_STRING string, const char * newbit) { ANTLR3_UINT32 len; len = (ANTLR3_UINT32)strlen(newbit); if (string->size < (string->len + len + 1)) { pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(string->len + len + 1)); if (newAlloc == NULL) { return NULL; } string->chars = newAlloc; string->size = string->len + len + 1; } /* Note we copy one more byte than the strlen in order to get the trailing */ ANTLR3_MEMMOVE((void *)(string->chars + string->len), newbit, (ANTLR3_UINT32)(len+1)); string->len += len; return string->chars; } static pANTLR3_UINT8 appendUTF16_8 (pANTLR3_STRING string, const char * newbit) { ANTLR3_UINT32 len; pANTLR3_UINT16 apPoint; ANTLR3_UINT32 count; len = (ANTLR3_UINT32)strlen(newbit); if (string->size < (string->len + len + 1)) { pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)((sizeof(ANTLR3_UINT16)*(string->len + len + 1)))); if (newAlloc == NULL) { return NULL; } string->chars = newAlloc; string->size = string->len + len + 1; } apPoint = ((pANTLR3_UINT16)string->chars) + string->len; string->len += len; for (count = 0; count < len; count++) { *apPoint++ = *(newbit + count); } *apPoint = '\0'; return string->chars; } static pANTLR3_UINT8 appendUTF16_UTF16 (pANTLR3_STRING string, const char * newbit) { ANTLR3_UINT32 len; pANTLR3_UINT16 in; /** First, determine the length of the input string */ in = (pANTLR3_UINT16)newbit; len = 0; while (*in++ != '\0') { len++; } if (string->size < (string->len + len + 1)) { pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)( sizeof(ANTLR3_UINT16) *(string->len + len + 1) )); if (newAlloc == NULL) { return NULL; } string->chars = newAlloc; string->size = string->len + len + 1; } /* Note we copy one more byte than the strlen in order to get the trailing delimiter */ ANTLR3_MEMMOVE((void *)(((pANTLR3_UINT16)string->chars) + string->len), newbit, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(len+1))); string->len += len; return string->chars; } static pANTLR3_UINT8 set8 (pANTLR3_STRING string, const char * chars) { ANTLR3_UINT32 len; len = (ANTLR3_UINT32)strlen(chars); if (string->size < len + 1) { pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(len + 1)); if (newAlloc == NULL) { return NULL; } string->chars = newAlloc; string->size = len + 1; } /* Note we copy one more byte than the strlen in order to get the trailing '\0' */ ANTLR3_MEMMOVE((void *)(string->chars), chars, (ANTLR3_UINT32)(len+1)); string->len = len; return string->chars; } static pANTLR3_UINT8 setUTF16_8 (pANTLR3_STRING string, const char * chars) { ANTLR3_UINT32 len; ANTLR3_UINT32 count; pANTLR3_UINT16 apPoint; len = (ANTLR3_UINT32)strlen(chars); if (string->size < len + 1) { pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(len + 1))); if (newAlloc == NULL) { return NULL; } string->chars = newAlloc; string->size = len + 1; } apPoint = ((pANTLR3_UINT16)string->chars); string->len = len; for (count = 0; count < string->len; count++) { *apPoint++ = *(chars + count); } *apPoint = '\0'; return string->chars; } static pANTLR3_UINT8 setUTF16_UTF16 (pANTLR3_STRING string, const char * chars) { ANTLR3_UINT32 len; pANTLR3_UINT16 in; /** First, determine the length of the input string */ in = (pANTLR3_UINT16)chars; len = 0; while (*in++ != '\0') { len++; } if (string->size < len + 1) { pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(len + 1))); if (newAlloc == NULL) { return NULL; } string->chars = newAlloc; string->size = len + 1; } /* Note we copy one more byte than the strlen in order to get the trailing '\0' */ ANTLR3_MEMMOVE((void *)(string->chars), chars, (ANTLR3_UINT32)((len+1) * sizeof(ANTLR3_UINT16))); string->len = len; return string->chars; } static pANTLR3_UINT8 addc8 (pANTLR3_STRING string, ANTLR3_UINT32 c) { if (string->size < string->len + 2) { pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(string->len + 2)); if (newAlloc == NULL) { return NULL; } string->chars = newAlloc; string->size = string->len + 2; } *(string->chars + string->len) = (ANTLR3_UINT8)c; *(string->chars + string->len + 1) = '\0'; string->len++; return string->chars; } static pANTLR3_UINT8 addcUTF16 (pANTLR3_STRING string, ANTLR3_UINT32 c) { pANTLR3_UINT16 ptr; if (string->size < string->len + 2) { pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16) * (string->len + 2))); if (newAlloc == NULL) { return NULL; } string->chars = newAlloc; string->size = string->len + 2; } ptr = (pANTLR3_UINT16)(string->chars); *(ptr + string->len) = (ANTLR3_UINT16)c; *(ptr + string->len + 1) = '\0'; string->len++; return string->chars; } static pANTLR3_UINT8 addi8 (pANTLR3_STRING string, ANTLR3_INT32 i) { ANTLR3_UINT8 newbit[32]; sprintf((char *)newbit, "%d", i); return string->append8(string, (const char *)newbit); } static pANTLR3_UINT8 addiUTF16 (pANTLR3_STRING string, ANTLR3_INT32 i) { ANTLR3_UINT8 newbit[32]; sprintf((char *)newbit, "%d", i); return string->append8(string, (const char *)newbit); } static pANTLR3_UINT8 inserti8 (pANTLR3_STRING string, ANTLR3_UINT32 point, ANTLR3_INT32 i) { ANTLR3_UINT8 newbit[32]; sprintf((char *)newbit, "%d", i); return string->insert8(string, point, (const char *)newbit); } static pANTLR3_UINT8 insertiUTF16 (pANTLR3_STRING string, ANTLR3_UINT32 point, ANTLR3_INT32 i) { ANTLR3_UINT8 newbit[32]; sprintf((char *)newbit, "%d", i); return string->insert8(string, point, (const char *)newbit); } static pANTLR3_UINT8 insert8 (pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit) { ANTLR3_UINT32 len; if (point >= string->len) { return string->append(string, newbit); } len = (ANTLR3_UINT32)strlen(newbit); if (len == 0) { return string->chars; } if (string->size < (string->len + len + 1)) { pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(string->len + len + 1)); if (newAlloc == NULL) { return NULL; } string->chars = newAlloc; string->size = string->len + len + 1; } /* Move the characters we are inserting before, including the delimiter */ ANTLR3_MEMMOVE((void *)(string->chars + point + len), (void *)(string->chars + point), (ANTLR3_UINT32)(string->len - point + 1)); /* Note we copy the exact number of bytes */ ANTLR3_MEMMOVE((void *)(string->chars + point), newbit, (ANTLR3_UINT32)(len)); string->len += len; return string->chars; } static pANTLR3_UINT8 insertUTF16_8 (pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit) { ANTLR3_UINT32 len; ANTLR3_UINT32 count; pANTLR3_UINT16 inPoint; if (point >= string->len) { return string->append8(string, newbit); } len = (ANTLR3_UINT32)strlen(newbit); if (len == 0) { return string->chars; } if (string->size < (string->len + len + 1)) { pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(string->len + len + 1))); if (newAlloc == NULL) { return NULL; } string->chars = newAlloc; string->size = string->len + len + 1; } /* Move the characters we are inserting before, including the delimiter */ ANTLR3_MEMMOVE((void *)(((pANTLR3_UINT16)string->chars) + point + len), (void *)(((pANTLR3_UINT16)string->chars) + point), (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(string->len - point + 1))); string->len += len; inPoint = ((pANTLR3_UINT16)(string->chars))+point; for (count = 0; countchars; } static pANTLR3_UINT8 insertUTF16_UTF16 (pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit) { ANTLR3_UINT32 len; pANTLR3_UINT16 in; if (point >= string->len) { return string->append(string, newbit); } /** First, determine the length of the input string */ in = (pANTLR3_UINT16)newbit; len = 0; while (*in++ != '\0') { len++; } if (len == 0) { return string->chars; } if (string->size < (string->len + len + 1)) { pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(string->len + len + 1))); if (newAlloc == NULL) { return NULL; } string->chars = newAlloc; string->size = string->len + len + 1; } /* Move the characters we are inserting before, including the delimiter */ ANTLR3_MEMMOVE((void *)(((pANTLR3_UINT16)string->chars) + point + len), (void *)(((pANTLR3_UINT16)string->chars) + point), (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(string->len - point + 1))); /* Note we copy the exact number of characters */ ANTLR3_MEMMOVE((void *)(((pANTLR3_UINT16)string->chars) + point), newbit, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(len))); string->len += len; return string->chars; } static pANTLR3_UINT8 setS (pANTLR3_STRING string, pANTLR3_STRING chars) { return string->set(string, (const char *)(chars->chars)); } static pANTLR3_UINT8 appendS (pANTLR3_STRING string, pANTLR3_STRING newbit) { /* We may be passed an empty string, in which case we just return the current pointer */ if (newbit == NULL || newbit->len == 0 || newbit->size == 0 || newbit->chars == NULL) { return string->chars; } else { return string->append(string, (const char *)(newbit->chars)); } } static pANTLR3_UINT8 insertS (pANTLR3_STRING string, ANTLR3_UINT32 point, pANTLR3_STRING newbit) { return string->insert(string, point, (const char *)(newbit->chars)); } /* Function that compares the text of a string to the supplied * 8 bit character string and returns a result a la strcmp() */ static ANTLR3_UINT32 compare8 (pANTLR3_STRING string, const char * compStr) { return strcmp((const char *)(string->chars), compStr); } /* Function that compares the text of a string with the supplied character string * (which is assumed to be in the same encoding as the string itself) and returns a result * a la strcmp() */ static ANTLR3_UINT32 compareUTF16_8 (pANTLR3_STRING string, const char * compStr) { pANTLR3_UINT16 ourString; ANTLR3_UINT32 charDiff; ourString = (pANTLR3_UINT16)(string->chars); while (((ANTLR3_UCHAR)(*ourString) != '\0') && ((ANTLR3_UCHAR)(*compStr) != '\0')) { charDiff = *ourString - *compStr; if (charDiff != 0) { return charDiff; } ourString++; compStr++; } /* At this point, one of the strings was terminated */ return (ANTLR3_UINT32)((ANTLR3_UCHAR)(*ourString) - (ANTLR3_UCHAR)(*compStr)); } /* Function that compares the text of a string with the supplied character string * (which is assumed to be in the same encoding as the string itself) and returns a result * a la strcmp() */ static ANTLR3_UINT32 compareUTF16_UTF16 (pANTLR3_STRING string, const char * compStr8) { pANTLR3_UINT16 ourString; pANTLR3_UINT16 compStr; ANTLR3_UINT32 charDiff; ourString = (pANTLR3_UINT16)(string->chars); compStr = (pANTLR3_UINT16)(compStr8); while (((ANTLR3_UCHAR)(*ourString) != '\0') && ((ANTLR3_UCHAR)(*((pANTLR3_UINT16)compStr)) != '\0')) { charDiff = *ourString - *compStr; if (charDiff != 0) { return charDiff; } ourString++; compStr++; } /* At this point, one of the strings was terminated */ return (ANTLR3_UINT32)((ANTLR3_UCHAR)(*ourString) - (ANTLR3_UCHAR)(*compStr)); } /* Function that compares the text of a string with the supplied string * (which is assumed to be in the same encoding as the string itself) and returns a result * a la strcmp() */ static ANTLR3_UINT32 compareS (pANTLR3_STRING string, pANTLR3_STRING compStr) { return string->compare(string, (const char *)compStr->chars); } /* Function that returns the character indexed at the supplied * offset as a 32 bit character. */ static ANTLR3_UCHAR charAt8 (pANTLR3_STRING string, ANTLR3_UINT32 offset) { if (offset > string->len) { return (ANTLR3_UCHAR)'\0'; } else { return (ANTLR3_UCHAR)(*(string->chars + offset)); } } /* Function that returns the character indexed at the supplied * offset as a 32 bit character. */ static ANTLR3_UCHAR charAtUTF16 (pANTLR3_STRING string, ANTLR3_UINT32 offset) { if (offset > string->len) { return (ANTLR3_UCHAR)'\0'; } else { return (ANTLR3_UCHAR)(*((pANTLR3_UINT16)(string->chars) + offset)); } } /* Function that returns a substring of the supplied string a la .subString(s,e) * in java runtimes. */ static pANTLR3_STRING subString8 (pANTLR3_STRING string, ANTLR3_UINT32 startIndex, ANTLR3_UINT32 endIndex) { pANTLR3_STRING newStr; if (endIndex > string->len) { endIndex = string->len + 1; } newStr = string->factory->newPtr(string->factory, string->chars + startIndex, endIndex - startIndex); return newStr; } /* Returns a substring of the supplied string a la .subString(s,e) * in java runtimes. */ static pANTLR3_STRING subStringUTF16 (pANTLR3_STRING string, ANTLR3_UINT32 startIndex, ANTLR3_UINT32 endIndex) { pANTLR3_STRING newStr; if (endIndex > string->len) { endIndex = string->len + 1; } newStr = string->factory->newPtr(string->factory, (pANTLR3_UINT8)((pANTLR3_UINT16)(string->chars) + startIndex), endIndex - startIndex); return newStr; } /* Function that can convert the characters in the string to an integer */ static ANTLR3_INT32 toInt32_8 (struct ANTLR3_STRING_struct * string) { return atoi((const char *)(string->chars)); } /* Function that can convert the characters in the string to an integer */ static ANTLR3_INT32 toInt32_UTF16 (struct ANTLR3_STRING_struct * string) { pANTLR3_UINT16 input; ANTLR3_INT32 value; ANTLR3_BOOLEAN negate; value = 0; input = (pANTLR3_UINT16)(string->chars); negate = ANTLR3_FALSE; if (*input == (ANTLR3_UCHAR)'-') { negate = ANTLR3_TRUE; input++; } else if (*input == (ANTLR3_UCHAR)'+') { input++; } while (*input != '\0' && isdigit(*input)) { value = value * 10; value += ((ANTLR3_UINT32)(*input) - (ANTLR3_UINT32)'0'); input++; } return negate ? -value : value; } /* Function that returns a pointer to an 8 bit version of the string, * which in this case is just the string as this is * 8 bit encodiing anyway. */ static pANTLR3_STRING to8_8 (pANTLR3_STRING string) { return string; } /* Function that returns an 8 bit version of the string, * which in this case is returning all the UTF16 characters * narrowed back into 8 bits, with characters that are too large * replaced with '_' */ static pANTLR3_STRING to8_UTF16 (pANTLR3_STRING string) { pANTLR3_STRING newStr; ANTLR3_UINT32 i; /* Create a new 8 bit string */ newStr = newRaw8(string->factory); if (newStr == NULL) { return NULL; } /* Always add one more byte for a terminator */ newStr->chars = (pANTLR3_UINT8) ANTLR3_MALLOC((size_t)(string->len + 1)); if (newStr->chars != NULL) { newStr->size = string->len + 1; newStr->len = string->len; /* Now copy each UTF16 charActer , making it an 8 bit character of * some sort. */ for (i=0; ilen; i++) { ANTLR3_UCHAR c; c = *(((pANTLR3_UINT16)(string->chars)) + i); *(newStr->chars + i) = (ANTLR3_UINT8)(c > 255 ? '_' : c); } /* Terminate */ *(newStr->chars + newStr->len) = '\0'; } return newStr; }