From 3b06ee0d381dc1be5f40ca98ad4278046d869d21 Mon Sep 17 00:00:00 2001
From: Andreas Baumann <mail@andreasbaumann.cc>
Date: Sun, 17 Nov 2019 20:57:39 +0100
Subject: checked in initial customized verison for Archlinux32

---
 include/utf8/utils/ascii.php      | 221 ++++++++++++++++++++
 include/utf8/utils/bad.php        | 430 ++++++++++++++++++++++++++++++++++++++
 include/utf8/utils/index.html     |   1 +
 include/utf8/utils/patterns.php   |  67 ++++++
 include/utf8/utils/position.php   | 171 +++++++++++++++
 include/utf8/utils/specials.php   | 131 ++++++++++++
 include/utf8/utils/unicode.php    | 241 +++++++++++++++++++++
 include/utf8/utils/validation.php | 186 +++++++++++++++++
 8 files changed, 1448 insertions(+)
 create mode 100644 include/utf8/utils/ascii.php
 create mode 100644 include/utf8/utils/bad.php
 create mode 100644 include/utf8/utils/index.html
 create mode 100644 include/utf8/utils/patterns.php
 create mode 100644 include/utf8/utils/position.php
 create mode 100644 include/utf8/utils/specials.php
 create mode 100644 include/utf8/utils/unicode.php
 create mode 100644 include/utf8/utils/validation.php

(limited to 'include/utf8/utils')

diff --git a/include/utf8/utils/ascii.php b/include/utf8/utils/ascii.php
new file mode 100644
index 0000000..af75b92
--- /dev/null
+++ b/include/utf8/utils/ascii.php
@@ -0,0 +1,221 @@
+<?php
+
+/**
+* Tools to help with ASCII in UTF-8
+* @version $Id: ascii.php,v 1.5 2006/10/16 20:38:12 harryf Exp $
+* @package utf8
+* @subpackage ascii
+*/
+
+/**
+* Tests whether a string contains only 7bit ASCII bytes.
+* You might use this to conditionally check whether a string
+* needs handling as UTF-8 or not, potentially offering performance
+* benefits by using the native PHP equivalent if it's just ASCII e.g.;
+*
+* <code>
+* if ( utf8_is_ascii($someString) ) {
+*     // It's just ASCII - use the native PHP version
+*     $someString = strtolower($someString);
+* } else {
+*     $someString = utf8_strtolower($someString);
+* }
+* </code>
+*
+* @param string
+* @return boolean TRUE if it's all ASCII
+* @package utf8
+* @subpackage ascii
+* @see utf8_is_ascii_ctrl
+*/
+function utf8_is_ascii($str)
+{
+	// Search for any bytes which are outside the ASCII range...
+	return (preg_match('/(?:[^\x00-\x7F])/', $str) !== 1);
+}
+
+/**
+* Tests whether a string contains only 7bit ASCII bytes with device
+* control codes omitted. The device control codes can be found on the
+* second table here: http://www.w3schools.com/tags/ref_ascii.asp
+*
+* @param string
+* @return boolean TRUE if it's all ASCII without device control codes
+* @package utf8
+* @subpackage ascii
+* @see utf8_is_ascii
+*/
+function utf8_is_ascii_ctrl($str)
+{
+	// Search for any bytes which are outside the ASCII range, or are device control codes
+	if (strlen($str) > 0)
+		return (preg_match('/[^\x09\x0A\x0D\x20-\x7E]/', $str) !== 1);
+
+	return false;
+}
+
+/**
+* Strip out all non-7bit ASCII bytes
+* If you need to transmit a string to system which you know can only
+* support 7bit ASCII, you could use this function.
+* @param string
+* @return string with non ASCII bytes removed
+* @package utf8
+* @subpackage ascii
+* @see utf8_strip_non_ascii_ctrl
+*/
+function utf8_strip_non_ascii($str)
+{
+	ob_start();
+
+	while (preg_match('/^([\x00-\x7F]+)|([^\x00-\x7F]+)/S', $str, $matches))
+	{
+		if (!isset($matches[2]))
+			echo $matches[0];
+
+		$str = substr($str, strlen($matches[0]));
+	}
+
+	$result = ob_get_contents();
+	ob_end_clean();
+
+	return $result;
+}
+
+/**
+* Strip out device control codes in the ASCII range
+* which are not permitted in XML. Note that this leaves
+* multi-byte characters untouched - it only removes device
+* control codes
+* @see http://hsivonen.iki.fi/producing-xml/#controlchar
+* @param string
+* @return string control codes removed
+*/
+function utf8_strip_ascii_ctrl($str)
+{
+	ob_start();
+
+	while (preg_match('/^([^\x00-\x08\x0B\x0C\x0E-\x1F\x7F]+)|([\x00-\x08\x0B\x0C\x0E-\x1F\x7F]+)/S', $str, $matches))
+	{
+		if (!isset($matches[2]))
+			echo $matches[0];
+
+		$str = substr($str, strlen($matches[0]));
+	}
+
+	$result = ob_get_contents();
+	ob_end_clean();
+
+	return $result;
+}
+
+/**
+* Strip out all non 7bit ASCII bytes and ASCII device control codes.
+* For a list of ASCII device control codes see the 2nd table here:
+* http://www.w3schools.com/tags/ref_ascii.asp
+*
+* @param string
+* @return boolean TRUE if it's all ASCII
+* @package utf8
+* @subpackage ascii
+*/
+function utf8_strip_non_ascii_ctrl($str)
+{
+	ob_start();
+
+	while (preg_match( '/^([\x09\x0A\x0D\x20-\x7E]+)|([^\x09\x0A\x0D\x20-\x7E]+)/S', $str, $matches))
+	{
+		if (!isset($matches[2]))
+			echo $matches[0];
+
+		$str = substr($str, strlen($matches[0]));
+	}
+
+	$result = ob_get_contents();
+	ob_end_clean();
+
+	return $result;
+}
+
+/**
+* Replace accented UTF-8 characters by unaccented ASCII-7 "equivalents".
+* The purpose of this function is to replace characters commonly found in Latin
+* alphabets with something more or less equivalent from the ASCII range. This can
+* be useful for converting a UTF-8 to something ready for a filename, for example.
+* Following the use of this function, you would probably also pass the string
+* through utf8_strip_non_ascii to clean out any other non-ASCII chars
+* Use the optional parameter to just deaccent lower ($case = -1) or upper ($case = 1)
+* letters. Default is to deaccent both cases ($case = 0)
+*
+* For a more complete implementation of transliteration, see the utf8_to_ascii package
+* available from the phputf8 project downloads:
+* http://prdownloads.sourceforge.net/phputf8
+*
+* @param string UTF-8 string
+* @param int (optional) -1 lowercase only, +1 uppercase only, 1 both cases
+* @param string UTF-8 with accented characters replaced by ASCII chars
+* @return string accented chars replaced with ascii equivalents
+* @author Andreas Gohr <andi@splitbrain.org>
+* @package utf8
+* @subpackage ascii
+*/
+function utf8_accents_to_ascii($str, $case=0)
+{
+	static $UTF8_LOWER_ACCENTS = null;
+	static $UTF8_UPPER_ACCENTS = null;
+
+	if($case <= 0)
+	{
+
+		if (is_null($UTF8_LOWER_ACCENTS))
+		{
+			$UTF8_LOWER_ACCENTS = array(
+				'à' => 'a', 'ô' => 'o', 'ď' => 'd', 'ḟ' => 'f', 'ë' => 'e', 'š' => 's', 'ơ' => 'o',
+				'ß' => 'ss', 'ă' => 'a', 'ř' => 'r', 'ț' => 't', 'ň' => 'n', 'ā' => 'a', 'ķ' => 'k',
+				'ŝ' => 's', 'ỳ' => 'y', 'ņ' => 'n', 'ĺ' => 'l', 'ħ' => 'h', 'ṗ' => 'p', 'ó' => 'o',
+				'ú' => 'u', 'ě' => 'e', 'é' => 'e', 'ç' => 'c', 'ẁ' => 'w', 'ċ' => 'c', 'õ' => 'o',
+				'ṡ' => 's', 'ø' => 'o', 'ģ' => 'g', 'ŧ' => 't', 'ș' => 's', 'ė' => 'e', 'ĉ' => 'c',
+				'ś' => 's', 'î' => 'i', 'ű' => 'u', 'ć' => 'c', 'ę' => 'e', 'ŵ' => 'w', 'ṫ' => 't',
+				'ū' => 'u', 'č' => 'c', 'ö' => 'oe', 'è' => 'e', 'ŷ' => 'y', 'ą' => 'a', 'ł' => 'l',
+				'ų' => 'u', 'ů' => 'u', 'ş' => 's', 'ğ' => 'g', 'ļ' => 'l', 'ƒ' => 'f', 'ž' => 'z',
+				'ẃ' => 'w', 'ḃ' => 'b', 'å' => 'a', 'ì' => 'i', 'ï' => 'i', 'ḋ' => 'd', 'ť' => 't',
+				'ŗ' => 'r', 'ä' => 'ae', 'í' => 'i', 'ŕ' => 'r', 'ê' => 'e', 'ü' => 'ue', 'ò' => 'o',
+				'ē' => 'e', 'ñ' => 'n', 'ń' => 'n', 'ĥ' => 'h', 'ĝ' => 'g', 'đ' => 'd', 'ĵ' => 'j',
+				'ÿ' => 'y', 'ũ' => 'u', 'ŭ' => 'u', 'ư' => 'u', 'ţ' => 't', 'ý' => 'y', 'ő' => 'o',
+				'â' => 'a', 'ľ' => 'l', 'ẅ' => 'w', 'ż' => 'z', 'ī' => 'i', 'ã' => 'a', 'ġ' => 'g',
+				'ṁ' => 'm', 'ō' => 'o', 'ĩ' => 'i', 'ù' => 'u', 'į' => 'i', 'ź' => 'z', 'á' => 'a',
+				'û' => 'u', 'þ' => 'th', 'ð' => 'dh', 'æ' => 'ae', 'µ' => 'u', 'ĕ' => 'e',
+				);
+		}
+
+		$str = str_replace(array_keys($UTF8_LOWER_ACCENTS), array_values($UTF8_LOWER_ACCENTS), $str);
+	}
+
+	if($case >= 0)
+	{
+		if (is_null($UTF8_UPPER_ACCENTS))
+		{
+			$UTF8_UPPER_ACCENTS = array(
+				'À' => 'A', 'Ô' => 'O', 'Ď' => 'D', 'Ḟ' => 'F', 'Ë' => 'E', 'Š' => 'S', 'Ơ' => 'O',
+				'Ă' => 'A', 'Ř' => 'R', 'Ț' => 'T', 'Ň' => 'N', 'Ā' => 'A', 'Ķ' => 'K',
+				'Ŝ' => 'S', 'Ỳ' => 'Y', 'Ņ' => 'N', 'Ĺ' => 'L', 'Ħ' => 'H', 'Ṗ' => 'P', 'Ó' => 'O',
+				'Ú' => 'U', 'Ě' => 'E', 'É' => 'E', 'Ç' => 'C', 'Ẁ' => 'W', 'Ċ' => 'C', 'Õ' => 'O',
+				'Ṡ' => 'S', 'Ø' => 'O', 'Ģ' => 'G', 'Ŧ' => 'T', 'Ș' => 'S', 'Ė' => 'E', 'Ĉ' => 'C',
+				'Ś' => 'S', 'Î' => 'I', 'Ű' => 'U', 'Ć' => 'C', 'Ę' => 'E', 'Ŵ' => 'W', 'Ṫ' => 'T',
+				'Ū' => 'U', 'Č' => 'C', 'Ö' => 'Oe', 'È' => 'E', 'Ŷ' => 'Y', 'Ą' => 'A', 'Ł' => 'L',
+				'Ų' => 'U', 'Ů' => 'U', 'Ş' => 'S', 'Ğ' => 'G', 'Ļ' => 'L', 'Ƒ' => 'F', 'Ž' => 'Z',
+				'Ẃ' => 'W', 'Ḃ' => 'B', 'Å' => 'A', 'Ì' => 'I', 'Ï' => 'I', 'Ḋ' => 'D', 'Ť' => 'T',
+				'Ŗ' => 'R', 'Ä' => 'Ae', 'Í' => 'I', 'Ŕ' => 'R', 'Ê' => 'E', 'Ü' => 'Ue', 'Ò' => 'O',
+				'Ē' => 'E', 'Ñ' => 'N', 'Ń' => 'N', 'Ĥ' => 'H', 'Ĝ' => 'G', 'Đ' => 'D', 'Ĵ' => 'J',
+				'Ÿ' => 'Y', 'Ũ' => 'U', 'Ŭ' => 'U', 'Ư' => 'U', 'Ţ' => 'T', 'Ý' => 'Y', 'Ő' => 'O',
+				'Â' => 'A', 'Ľ' => 'L', 'Ẅ' => 'W', 'Ż' => 'Z', 'Ī' => 'I', 'Ã' => 'A', 'Ġ' => 'G',
+				'Ṁ' => 'M', 'Ō' => 'O', 'Ĩ' => 'I', 'Ù' => 'U', 'Į' => 'I', 'Ź' => 'Z', 'Á' => 'A',
+				'Û' => 'U', 'Þ' => 'Th', 'Ð' => 'Dh', 'Æ' => 'Ae', 'Ĕ' => 'E',
+				);
+		}
+
+		$str = str_replace(array_keys($UTF8_UPPER_ACCENTS), array_values($UTF8_UPPER_ACCENTS),  $str);
+	}
+
+	return $str;
+}
diff --git a/include/utf8/utils/bad.php b/include/utf8/utils/bad.php
new file mode 100644
index 0000000..2704294
--- /dev/null
+++ b/include/utf8/utils/bad.php
@@ -0,0 +1,430 @@
+<?php
+
+/**
+* @version $Id: bad.php,v 1.2 2006/02/26 13:20:44 harryf Exp $
+* Tools for locating / replacing bad bytes in UTF-8 strings
+* The Original Code is Mozilla Communicator client code.
+* The Initial Developer of the Original Code is
+* Netscape Communications Corporation.
+* Portions created by the Initial Developer are Copyright (C) 1998
+* the Initial Developer. All Rights Reserved.
+* Ported to PHP by Henri Sivonen (http://hsivonen.iki.fi)
+* Slight modifications to fit with phputf8 library by Harry Fuecks (hfuecks gmail com)
+* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUTF8ToUnicode.cpp
+* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUnicodeToUTF8.cpp
+* @see http://hsivonen.iki.fi/php-utf8/
+* @package utf8
+* @subpackage bad
+* @see utf8_is_valid
+*/
+
+/**
+* Locates the first bad byte in a UTF-8 string returning it's
+* byte index in the string
+* PCRE Pattern to locate bad bytes in a UTF-8 string
+* Comes from W3 FAQ: Multilingual Forms
+* Note: modified to include full ASCII range including control chars
+* @see http://www.w3.org/International/questions/qa-forms-utf-8
+* @param string
+* @return mixed integer byte index or FALSE if no bad found
+* @package utf8
+* @subpackage bad
+*/
+function utf8_bad_find($str)
+{
+	$UTF8_BAD =
+		'([\x00-\x7F]'.                          # ASCII (including control chars)
+		'|[\xC2-\xDF][\x80-\xBF]'.               # Non-overlong 2-byte
+		'|\xE0[\xA0-\xBF][\x80-\xBF]'.           # Excluding overlongs
+		'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'.    # Straight 3-byte
+		'|\xED[\x80-\x9F][\x80-\xBF]'.           # Excluding surrogates
+		'|\xF0[\x90-\xBF][\x80-\xBF]{2}'.        # Planes 1-3
+		'|[\xF1-\xF3][\x80-\xBF]{3}'.            # Planes 4-15
+		'|\xF4[\x80-\x8F][\x80-\xBF]{2}'.        # Plane 16
+		'|(.{1}))';                              # Invalid byte
+	$pos = 0;
+	$badList = array();
+
+	while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches))
+	{
+		$bytes = strlen($matches[0]);
+
+		if (isset($matches[2]))
+			return $pos;
+
+		$pos += $bytes;
+		$str = substr($str,$bytes);
+	}
+
+	return false;
+}
+
+/**
+* Locates all bad bytes in a UTF-8 string and returns a list of their
+* byte index in the string
+* PCRE Pattern to locate bad bytes in a UTF-8 string
+* Comes from W3 FAQ: Multilingual Forms
+* Note: modified to include full ASCII range including control chars
+* @see http://www.w3.org/International/questions/qa-forms-utf-8
+* @param string
+* @return mixed array of integers or FALSE if no bad found
+* @package utf8
+* @subpackage bad
+*/
+function utf8_bad_findall($str)
+{
+	$UTF8_BAD =
+		'([\x00-\x7F]'.                          # ASCII (including control chars)
+		'|[\xC2-\xDF][\x80-\xBF]'.               # Non-overlong 2-byte
+		'|\xE0[\xA0-\xBF][\x80-\xBF]'.           # Excluding overlongs
+		'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'.    # Straight 3-byte
+		'|\xED[\x80-\x9F][\x80-\xBF]'.           # Excluding surrogates
+		'|\xF0[\x90-\xBF][\x80-\xBF]{2}'.        # Planes 1-3
+		'|[\xF1-\xF3][\x80-\xBF]{3}'.            # Planes 4-15
+		'|\xF4[\x80-\x8F][\x80-\xBF]{2}'.        # Plane 16
+		'|(.{1}))';                              # Invalid byte
+	$pos = 0;
+	$badList = array();
+
+	while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches))
+	{
+		$bytes = strlen($matches[0]);
+
+		if (isset($matches[2]))
+			$badList[] = $pos;
+
+		$pos += $bytes;
+		$str = substr($str,$bytes);
+	}
+
+	if (count($badList) > 0)
+		return $badList;
+
+	return false;
+}
+
+/**
+* Strips out any bad bytes from a UTF-8 string and returns the rest
+* PCRE Pattern to locate bad bytes in a UTF-8 string
+* Comes from W3 FAQ: Multilingual Forms
+* Note: modified to include full ASCII range including control chars
+* @see http://www.w3.org/International/questions/qa-forms-utf-8
+* @param string
+* @return string
+* @package utf8
+* @subpackage bad
+*/
+function utf8_bad_strip($original)
+{
+	return utf8_bad_replace($original, '');
+}
+
+/**
+* Replace bad bytes with an alternative character - ASCII character
+* recommended is replacement char
+* PCRE Pattern to locate bad bytes in a UTF-8 string
+* Comes from W3 FAQ: Multilingual Forms
+* Note: modified to include full ASCII range including control chars
+* @see http://www.w3.org/International/questions/qa-forms-utf-8
+* @param string to search
+* @param string to replace bad bytes with (defaults to '?') - use ASCII
+* @return string
+* @package utf8
+* @subpackage bad
+*/
+function utf8_bad_replace($original, $replace = '?') {
+	$result = '';
+
+	$strlen = strlen($original);
+	for ($i = 0; $i < $strlen;) {
+		$char = $original[$i++];
+		$byte = ord($char);
+
+		if ($byte < 0x80) $bytes = 0; // 1-bytes (00000000 - 01111111)
+		else if ($byte < 0xC0) { // 1-bytes (10000000 - 10111111)
+			$result .= $replace;
+			continue;
+		}
+		else if ($byte < 0xE0) $bytes = 1; // 2-bytes (11000000 - 11011111)
+		else if ($byte < 0xF0) $bytes = 2; // 3-bytes (11100000 - 11101111)
+		else if ($byte < 0xF8) $bytes = 3; // 4-bytes (11110000 - 11110111)
+		else if ($byte < 0xFC) $bytes = 4; // 5-bytes (11111000 - 11111011)
+		else if ($byte < 0xFE) $bytes = 5; // 6-bytes (11111100 - 11111101)
+		else { // Otherwise it's something invalid
+			$result .= $replace;
+			continue;
+		}
+
+		// Check our input actually has enough data
+		if ($i + $bytes > $strlen) {
+			$result .= $replace;
+			continue;
+		}
+
+		// If we've got this far then we have a multiple-byte character
+		for ($j = 0; $j < $bytes; $j++) {
+			$byte = $original[$i + $j];
+
+			$char .= $byte;
+			$byte = ord($byte);
+
+			// Every following byte must be 10000000 - 10111111
+			if ($byte < 0x80 || $byte > 0xBF) {
+				$result .= $replace;
+				continue 2;
+			}
+		}
+
+		$i += $bytes;
+		$result .= $char;
+	}
+
+	return $result;
+}
+
+/**
+* Return code from utf8_bad_identify() when a five octet sequence is detected.
+* Note: 5 octets sequences are valid UTF-8 but are not supported by Unicode so
+* do not represent a useful character
+* @see utf8_bad_identify
+* @package utf8
+* @subpackage bad
+*/
+define('UTF8_BAD_5OCTET', 1);
+
+/**
+* Return code from utf8_bad_identify() when a six octet sequence is detected.
+* Note: 6 octets sequences are valid UTF-8 but are not supported by Unicode so
+* do not represent a useful character
+* @see utf8_bad_identify
+* @package utf8
+* @subpackage bad
+*/
+define('UTF8_BAD_6OCTET', 2);
+
+/**
+* Return code from utf8_bad_identify().
+* Invalid octet for use as start of multi-byte UTF-8 sequence
+* @see utf8_bad_identify
+* @package utf8
+* @subpackage bad
+*/
+define('UTF8_BAD_SEQID', 3);
+
+/**
+* Return code from utf8_bad_identify().
+* From Unicode 3.1, non-shortest form is illegal
+* @see utf8_bad_identify
+* @package utf8
+* @subpackage bad
+*/
+define('UTF8_BAD_NONSHORT', 4);
+
+/**
+* Return code from utf8_bad_identify().
+* From Unicode 3.2, surrogate characters are illegal
+* @see utf8_bad_identify
+* @package utf8
+* @subpackage bad
+*/
+define('UTF8_BAD_SURROGATE', 5);
+
+/**
+* Return code from utf8_bad_identify().
+* Codepoints outside the Unicode range are illegal
+* @see utf8_bad_identify
+* @package utf8
+* @subpackage bad
+*/
+define('UTF8_BAD_UNIOUTRANGE', 6);
+
+/**
+* Return code from utf8_bad_identify().
+* Incomplete multi-octet sequence
+* Note: this is kind of a "catch-all"
+* @see utf8_bad_identify
+* @package utf8
+* @subpackage bad
+*/
+define('UTF8_BAD_SEQINCOMPLETE', 7);
+
+/**
+* Reports on the type of bad byte found in a UTF-8 string. Returns a
+* status code on the first bad byte found
+* @author <hsivonen@iki.fi>
+* @param string UTF-8 encoded string
+* @return mixed integer constant describing problem or FALSE if valid UTF-8
+* @see utf8_bad_explain
+* @see http://hsivonen.iki.fi/php-utf8/
+* @package utf8
+* @subpackage bad
+*/
+function utf8_bad_identify($str, &$i)
+{
+	$mState = 0;	// Cached expected number of octets after the current octet
+					// until the beginning of the next UTF8 character sequence
+	$mUcs4  = 0;	// Cached Unicode character
+	$mBytes = 1;	// Cached expected number of octets in the current sequence
+
+	$len = strlen($str);
+
+	for($i=0; $i < $len; $i++)
+	{
+		$in = ord($str{$i});
+
+		if ( $mState == 0)
+		{
+			// When mState is zero we expect either a US-ASCII character or a multi-octet sequence.
+			if (0 == (0x80 & ($in)))
+			{
+				// US-ASCII, pass straight through.
+				$mBytes = 1;
+			}
+			else if (0xC0 == (0xE0 & ($in)))
+			{
+				// First octet of 2 octet sequence
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 0x1F) << 6;
+				$mState = 1;
+				$mBytes = 2;
+			}
+			else if (0xE0 == (0xF0 & ($in)))
+			{
+				// First octet of 3 octet sequence
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 0x0F) << 12;
+				$mState = 2;
+				$mBytes = 3;
+			}
+			else if (0xF0 == (0xF8 & ($in)))
+			{
+				// First octet of 4 octet sequence
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 0x07) << 18;
+				$mState = 3;
+				$mBytes = 4;
+			}
+			else if (0xF8 == (0xFC & ($in)))
+			{
+				/* First octet of 5 octet sequence.
+				*
+				* This is illegal because the encoded codepoint must be either
+				* (a) not the shortest form or
+				* (b) outside the Unicode range of 0-0x10FFFF.
+				*/
+				return UTF8_BAD_5OCTET;
+			}
+			else if (0xFC == (0xFE & ($in)))
+			{
+				// First octet of 6 octet sequence, see comments for 5 octet sequence.
+				return UTF8_BAD_6OCTET;
+			}
+			else
+			{
+				// Current octet is neither in the US-ASCII range nor a legal first
+				// octet of a multi-octet sequence.
+				return UTF8_BAD_SEQID;
+			}
+		}
+		else
+		{
+			// When mState is non-zero, we expect a continuation of the multi-octet sequence
+			if (0x80 == (0xC0 & ($in)))
+			{
+				// Legal continuation.
+				$shift = ($mState - 1) * 6;
+				$tmp = $in;
+				$tmp = ($tmp & 0x0000003F) << $shift;
+				$mUcs4 |= $tmp;
+
+				/**
+				* End of the multi-octet sequence. mUcs4 now contains the final
+				* Unicode codepoint to be output
+				*/
+				if (0 == --$mState)
+				{
+					// From Unicode 3.1, non-shortest form is illegal
+					if (((2 == $mBytes) && ($mUcs4 < 0x0080)) ||
+					((3 == $mBytes) && ($mUcs4 < 0x0800)) ||
+					((4 == $mBytes) && ($mUcs4 < 0x10000)) )
+						return UTF8_BAD_NONSHORT;
+					else if (($mUcs4 & 0xFFFFF800) == 0xD800) // From Unicode 3.2, surrogate characters are illegal
+						return UTF8_BAD_SURROGATE;
+					else if ($mUcs4 > 0x10FFFF) // Codepoints outside the Unicode range are illegal
+						return UTF8_BAD_UNIOUTRANGE;
+
+					// Initialize UTF8 cache
+					$mState = 0;
+					$mUcs4  = 0;
+					$mBytes = 1;
+				}
+
+			}
+			else
+			{
+				// ((0xC0 & (*in) != 0x80) && (mState != 0))
+				// Incomplete multi-octet sequence.
+				$i--;
+				return UTF8_BAD_SEQINCOMPLETE;
+			}
+		}
+	}
+
+	// Incomplete multi-octet sequence
+	if ($mState != 0)
+	{
+		$i--;
+		return UTF8_BAD_SEQINCOMPLETE;
+	}
+
+	// No bad octets found
+	$i = null;
+	return false;
+}
+
+/**
+* Takes a return code from utf8_bad_identify() are returns a message
+* (in English) explaining what the problem is.
+* @param int return code from utf8_bad_identify
+* @return mixed string message or FALSE if return code unknown
+* @see utf8_bad_identify
+* @package utf8
+* @subpackage bad
+*/
+function utf8_bad_explain($code)
+{
+	switch ($code)
+	{
+		case UTF8_BAD_5OCTET:
+		return 'Five octet sequences are valid UTF-8 but are not supported by Unicode';
+		break;
+
+		case UTF8_BAD_6OCTET:
+		return 'Six octet sequences are valid UTF-8 but are not supported by Unicode';
+		break;
+
+		case UTF8_BAD_SEQID:
+		return 'Invalid octet for use as start of multi-byte UTF-8 sequence';
+		break;
+
+		case UTF8_BAD_NONSHORT:
+		return 'From Unicode 3.1, non-shortest form is illegal';
+		break;
+
+		case UTF8_BAD_SURROGATE:
+		return 'From Unicode 3.2, surrogate characters are illegal';
+		break;
+
+		case UTF8_BAD_UNIOUTRANGE:
+		return 'Codepoints outside the Unicode range are illegal';
+		break;
+
+		case UTF8_BAD_SEQINCOMPLETE:
+		return 'Incomplete multi-octet sequence';
+		break;
+	}
+
+	trigger_error('Unknown error code: '.$code, E_USER_WARNING);
+
+	return false;
+}
diff --git a/include/utf8/utils/index.html b/include/utf8/utils/index.html
new file mode 100644
index 0000000..89337b2
--- /dev/null
+++ b/include/utf8/utils/index.html
@@ -0,0 +1 @@
+<html><head><title>.</title></head><body>.</body></html>
diff --git a/include/utf8/utils/patterns.php b/include/utf8/utils/patterns.php
new file mode 100644
index 0000000..5a85a4f
--- /dev/null
+++ b/include/utf8/utils/patterns.php
@@ -0,0 +1,67 @@
+<?php
+
+/**
+* PCRE Regular expressions for UTF-8. Note this file is not actually used by
+* the rest of the library but these regular expressions can be useful to have
+* available.
+* @version $Id: patterns.php,v 1.1 2006/02/25 14:20:02 harryf Exp $
+* @see http://www.w3.org/International/questions/qa-forms-utf-8
+* @package utf8
+* @subpackage patterns
+*/
+
+/**
+* PCRE Pattern to check a UTF-8 string is valid
+* Comes from W3 FAQ: Multilingual Forms
+* Note: modified to include full ASCII range including control chars
+* @see http://www.w3.org/International/questions/qa-forms-utf-8
+* @package utf8
+* @subpackage patterns
+*/
+$UTF8_VALID = '^('.
+	'[\x00-\x7F]'.                          # ASCII (including control chars)
+	'|[\xC2-\xDF][\x80-\xBF]'.              # Non-overlong 2-byte
+	'|\xE0[\xA0-\xBF][\x80-\xBF]'.          # Excluding overlongs
+	'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'.   # Straight 3-byte
+	'|\xED[\x80-\x9F][\x80-\xBF]'.          # Excluding surrogates
+	'|\xF0[\x90-\xBF][\x80-\xBF]{2}'.       # Planes 1-3
+	'|[\xF1-\xF3][\x80-\xBF]{3}'.           # Planes 4-15
+	'|\xF4[\x80-\x8F][\x80-\xBF]{2}'.       # Plane 16
+	')*$';
+
+/**
+* PCRE Pattern to match single UTF-8 characters
+* Comes from W3 FAQ: Multilingual Forms
+* Note: modified to include full ASCII range including control chars
+* @see http://www.w3.org/International/questions/qa-forms-utf-8
+* @package utf8
+* @subpackage patterns
+*/
+$UTF8_MATCH =
+	'([\x00-\x7F])'.                          # ASCII (including control chars)
+	'|([\xC2-\xDF][\x80-\xBF])'.              # Non-overlong 2-byte
+	'|(\xE0[\xA0-\xBF][\x80-\xBF])'.          # Excluding overlongs
+	'|([\xE1-\xEC\xEE\xEF][\x80-\xBF]{2})'.   # Straight 3-byte
+	'|(\xED[\x80-\x9F][\x80-\xBF])'.          # Excluding surrogates
+	'|(\xF0[\x90-\xBF][\x80-\xBF]{2})'.       # Planes 1-3
+	'|([\xF1-\xF3][\x80-\xBF]{3})'.           # Planes 4-15
+	'|(\xF4[\x80-\x8F][\x80-\xBF]{2})';       # Plane 16
+
+/**
+* PCRE Pattern to locate bad bytes in a UTF-8 string
+* Comes from W3 FAQ: Multilingual Forms
+* Note: modified to include full ASCII range including control chars
+* @see http://www.w3.org/International/questions/qa-forms-utf-8
+* @package utf8
+* @subpackage patterns
+*/
+$UTF8_BAD =
+	'([\x00-\x7F]'.                          # ASCII (including control chars)
+	'|[\xC2-\xDF][\x80-\xBF]'.               # Non-overlong 2-byte
+	'|\xE0[\xA0-\xBF][\x80-\xBF]'.           # Excluding overlongs
+	'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'.    # Straight 3-byte
+	'|\xED[\x80-\x9F][\x80-\xBF]'.           # Excluding surrogates
+	'|\xF0[\x90-\xBF][\x80-\xBF]{2}'.        # Planes 1-3
+	'|[\xF1-\xF3][\x80-\xBF]{3}'.            # Planes 4-15
+	'|\xF4[\x80-\x8F][\x80-\xBF]{2}'.        # Plane 16
+	'|(.{1}))';                              # Invalid byte
diff --git a/include/utf8/utils/position.php b/include/utf8/utils/position.php
new file mode 100644
index 0000000..7c62d10
--- /dev/null
+++ b/include/utf8/utils/position.php
@@ -0,0 +1,171 @@
+<?php
+
+/**
+* Locate a byte index given a UTF-8 character index
+* @version $Id: position.php,v 1.1 2006/10/01 00:01:31 harryf Exp $
+* @package utf8
+* @subpackage position
+*/
+
+/**
+* Given a string and a character index in the string, in
+* terms of the UTF-8 character position, returns the byte
+* index of that character. Can be useful when you want to
+* PHP's native string functions but we warned, locating
+* the byte can be expensive
+* Takes variable number of parameters - first must be
+* the search string then 1 to n UTF-8 character positions
+* to obtain byte indexes for - it is more efficient to search
+* the string for multiple characters at once, than make
+* repeated calls to this function
+*
+* @author Chris Smith<chris@jalakai.co.uk>
+* @param string string to locate index in
+* @param int (n times)
+* @return mixed - int if only one input int, array if more
+* @return boolean TRUE if it's all ASCII
+* @package utf8
+* @subpackage position
+*/
+function utf8_byte_position()
+{
+	$args = func_get_args();
+	$str =& array_shift($args);
+
+	if (!is_string($str))
+		return false;
+
+	$result = array();
+	$prev = array(0, 0); // Trivial byte index, character offset pair
+	$i = utf8_locate_next_chr($str, 300); // Use a short piece of str to estimate bytes per character. $i (& $j) -> byte indexes into $str
+	$c = strlen(utf8_decode(substr($str, 0, $i))); // $c -> character offset into $str
+
+	// Deal with arguments from lowest to highest
+	sort($args);
+
+	foreach ($args as $offset)
+	{
+		// Sanity checks FIXME
+
+		// 0 is an easy check
+		if ($offset == 0)
+		{
+			$result[] = 0; continue;
+		}
+
+		// Ensure no endless looping
+		$safety_valve = 50;
+
+		do
+		{
+			if (($c - $prev[1]) == 0)
+			{
+				// Hack: gone past end of string
+				$error = 0;
+				$i = strlen($str);
+				break;
+			}
+
+			$j = $i + (int)(($offset-$c) * ($i - $prev[0]) / ($c - $prev[1]));
+			$j = utf8_locate_next_chr($str, $j); // Correct to utf8 character boundary
+			$prev = array($i,$c); // Save the index, offset for use next iteration
+
+			if ($j > $i)
+				$c += strlen(utf8_decode(substr($str, $i, $j-$i))); // Determine new character offset
+			else
+				$c -= strlen(utf8_decode(substr($str, $j, $i-$j))); // Ditto
+
+			$error = abs($c-$offset);
+			$i = $j; // Ready for next time around
+		}
+		while (($error > 7) && --$safety_valve); // From 7 it is faster to iterate over the string
+
+		if ($error && $error <= 7)
+		{
+			if ($c < $offset)
+			{
+				// Move up
+				while ($error--)
+					$i = utf8_locate_next_chr($str, ++$i);
+			}
+			else
+			{
+				// Move down
+				while ($error--)
+					$i = utf8_locate_current_chr($str, --$i);
+			}
+
+			// Ready for next arg
+			$c = $offset;
+		}
+
+		$result[] = $i;
+	}
+
+	if (count($result) == 1)
+		return $result[0];
+
+	return $result;
+}
+
+/**
+* Given a string and any byte index, returns the byte index
+* of the start of the current UTF-8 character, relative to supplied
+* position. If the current character begins at the same place as the
+* supplied byte index, that byte index will be returned. Otherwise
+* this function will step backwards, looking for the index where
+* curent UTF-8 character begins
+* @author Chris Smith<chris@jalakai.co.uk>
+* @param string
+* @param int byte index in the string
+* @return int byte index of start of next UTF-8 character
+* @package utf8
+* @subpackage position
+*/
+function utf8_locate_current_chr( &$str, $idx )
+{
+	if ($idx <= 0)
+		return 0;
+
+	$limit = strlen($str);
+	if ($idx >= $limit)
+		return $limit;
+
+	// Binary value for any byte after the first in a multi-byte UTF-8 character
+	// will be like 10xxxxxx so & 0xC0 can be used to detect this kind
+	// of byte - assuming well formed UTF-8
+	while ($idx && ((ord($str[$idx]) & 0xC0) == 0x80))
+		$idx--;
+
+	return $idx;
+}
+
+/**
+* Given a string and any byte index, returns the byte index
+* of the start of the next UTF-8 character, relative to supplied
+* position. If the next character begins at the same place as the
+* supplied byte index, that byte index will be returned.
+* @author Chris Smith<chris@jalakai.co.uk>
+* @param string
+* @param int byte index in the string
+* @return int byte index of start of next UTF-8 character
+* @package utf8
+* @subpackage position
+*/
+function utf8_locate_next_chr(&$str, $idx)
+{
+	if ($idx <= 0)
+		return 0;
+
+	$limit = strlen($str);
+	if ($idx >= $limit)
+		return $limit;
+
+	// Binary value for any byte after the first in a multi-byte UTF-8 character
+	// will be like 10xxxxxx so & 0xC0 can be used to detect this kind
+	// of byte - assuming well formed UTF-8
+	while (($idx < $limit) && ((ord($str[$idx]) & 0xC0) == 0x80))
+		$idx++;
+
+	return $idx;
+}
diff --git a/include/utf8/utils/specials.php b/include/utf8/utils/specials.php
new file mode 100644
index 0000000..69219dc
--- /dev/null
+++ b/include/utf8/utils/specials.php
@@ -0,0 +1,131 @@
+<?php
+
+/**
+* Utilities for processing "special" characters in UTF-8. "Special" largely means anything which would
+* be regarded as a non-word character, like ASCII control characters and punctuation. This has a "Roman"
+* bias - it would be unaware of modern Chinese "punctuation" characters for example.
+* Note: requires utils/unicode.php to be loaded
+* @version $Id: specials.php,v 1.2 2006/10/16 21:13:59 harryf Exp $
+* @package utf8
+* @subpackage utils
+* @see utf8_is_valid
+*/
+
+/**
+* Used internally. Builds a PCRE pattern from the $UTF8_SPECIAL_CHARS
+* array defined in this file
+* The $UTF8_SPECIAL_CHARS should contain all special characters (non-letter/non-digit)
+* defined in the various local charsets - it's not a complete list of
+* non-alphanum characters in UTF-8. It's not perfect but should match most
+* cases of special chars.
+* This function adds the control chars 0x00 to 0x19 to the array of
+* special chars (they are not included in $UTF8_SPECIAL_CHARS)
+* @package utf8
+* @subpackage utils
+* @return string
+* @see utf8_from_unicode
+* @see utf8_is_word_chars
+* @see utf8_strip_specials
+*/
+function utf8_specials_pattern()
+{
+	static $pattern = null;
+
+	if (!$pattern)
+	{
+		$UTF8_SPECIAL_CHARS = array(
+			0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023,
+			0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c,
+			0x002f,         0x003b, 0x003c, 0x003d, 0x003e, 0x003f, 0x0040, 0x005b,
+			0x005c, 0x005d, 0x005e,         0x0060, 0x007b, 0x007c, 0x007d, 0x007e,
+			0x007f, 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088,
+			0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, 0x0090, 0x0091, 0x0092,
+			0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c,
+			0x009d, 0x009e, 0x009f, 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6,
+			0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, 0x00b0,
+			0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba,
+			0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, 0x00d7, 0x00f7, 0x02c7, 0x02d8, 0x02d9,
+			0x02da, 0x02db, 0x02dc, 0x02dd, 0x0300, 0x0301, 0x0303, 0x0309, 0x0323, 0x0384,
+			0x0385, 0x0387, 0x03b2, 0x03c6, 0x03d1, 0x03d2, 0x03d5, 0x03d6, 0x05b0, 0x05b1,
+			0x05b2, 0x05b3, 0x05b4, 0x05b5, 0x05b6, 0x05b7, 0x05b8, 0x05b9, 0x05bb, 0x05bc,
+			0x05bd, 0x05be, 0x05bf, 0x05c0, 0x05c1, 0x05c2, 0x05c3, 0x05f3, 0x05f4, 0x060c,
+			0x061b, 0x061f, 0x0640, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f, 0x0650, 0x0651,
+			0x0652, 0x066a, 0x0e3f, 0x200c, 0x200d, 0x200e, 0x200f, 0x2013, 0x2014, 0x2015,
+			0x2017, 0x2018, 0x2019, 0x201a, 0x201c, 0x201d, 0x201e, 0x2020, 0x2021, 0x2022,
+			0x2026, 0x2030, 0x2032, 0x2033, 0x2039, 0x203a, 0x2044, 0x20a7, 0x20aa, 0x20ab,
+			0x20ac, 0x2116, 0x2118, 0x2122, 0x2126, 0x2135, 0x2190, 0x2191, 0x2192, 0x2193,
+			0x2194, 0x2195, 0x21b5, 0x21d0, 0x21d1, 0x21d2, 0x21d3, 0x21d4, 0x2200, 0x2202,
+			0x2203, 0x2205, 0x2206, 0x2207, 0x2208, 0x2209, 0x220b, 0x220f, 0x2211, 0x2212,
+			0x2215, 0x2217, 0x2219, 0x221a, 0x221d, 0x221e, 0x2220, 0x2227, 0x2228, 0x2229,
+			0x222a, 0x222b, 0x2234, 0x223c, 0x2245, 0x2248, 0x2260, 0x2261, 0x2264, 0x2265,
+			0x2282, 0x2283, 0x2284, 0x2286, 0x2287, 0x2295, 0x2297, 0x22a5, 0x22c5, 0x2310,
+			0x2320, 0x2321, 0x2329, 0x232a, 0x2469, 0x2500, 0x2502, 0x250c, 0x2510, 0x2514,
+			0x2518, 0x251c, 0x2524, 0x252c, 0x2534, 0x253c, 0x2550, 0x2551, 0x2552, 0x2553,
+			0x2554, 0x2555, 0x2556, 0x2557, 0x2558, 0x2559, 0x255a, 0x255b, 0x255c, 0x255d,
+			0x255e, 0x255f, 0x2560, 0x2561, 0x2562, 0x2563, 0x2564, 0x2565, 0x2566, 0x2567,
+			0x2568, 0x2569, 0x256a, 0x256b, 0x256c, 0x2580, 0x2584, 0x2588, 0x258c, 0x2590,
+			0x2591, 0x2592, 0x2593, 0x25a0, 0x25b2, 0x25bc, 0x25c6, 0x25ca, 0x25cf, 0x25d7,
+			0x2605, 0x260e, 0x261b, 0x261e, 0x2660, 0x2663, 0x2665, 0x2666, 0x2701, 0x2702,
+			0x2703, 0x2704, 0x2706, 0x2707, 0x2708, 0x2709, 0x270c, 0x270d, 0x270e, 0x270f,
+			0x2710, 0x2711, 0x2712, 0x2713, 0x2714, 0x2715, 0x2716, 0x2717, 0x2718, 0x2719,
+			0x271a, 0x271b, 0x271c, 0x271d, 0x271e, 0x271f, 0x2720, 0x2721, 0x2722, 0x2723,
+			0x2724, 0x2725, 0x2726, 0x2727, 0x2729, 0x272a, 0x272b, 0x272c, 0x272d, 0x272e,
+			0x272f, 0x2730, 0x2731, 0x2732, 0x2733, 0x2734, 0x2735, 0x2736, 0x2737, 0x2738,
+			0x2739, 0x273a, 0x273b, 0x273c, 0x273d, 0x273e, 0x273f, 0x2740, 0x2741, 0x2742,
+			0x2743, 0x2744, 0x2745, 0x2746, 0x2747, 0x2748, 0x2749, 0x274a, 0x274b, 0x274d,
+			0x274f, 0x2750, 0x2751, 0x2752, 0x2756, 0x2758, 0x2759, 0x275a, 0x275b, 0x275c,
+			0x275d, 0x275e, 0x2761, 0x2762, 0x2763, 0x2764, 0x2765, 0x2766, 0x2767, 0x277f,
+			0x2789, 0x2793, 0x2794, 0x2798, 0x2799, 0x279a, 0x279b, 0x279c, 0x279d, 0x279e,
+			0x279f, 0x27a0, 0x27a1, 0x27a2, 0x27a3, 0x27a4, 0x27a5, 0x27a6, 0x27a7, 0x27a8,
+			0x27a9, 0x27aa, 0x27ab, 0x27ac, 0x27ad, 0x27ae, 0x27af, 0x27b1, 0x27b2, 0x27b3,
+			0x27b4, 0x27b5, 0x27b6, 0x27b7, 0x27b8, 0x27b9, 0x27ba, 0x27bb, 0x27bc, 0x27bd,
+			0x27be, 0xf6d9, 0xf6da, 0xf6db, 0xf8d7, 0xf8d8, 0xf8d9, 0xf8da, 0xf8db, 0xf8dc,
+			0xf8dd, 0xf8de, 0xf8df, 0xf8e0, 0xf8e1, 0xf8e2, 0xf8e3, 0xf8e4, 0xf8e5, 0xf8e6,
+			0xf8e7, 0xf8e8, 0xf8e9, 0xf8ea, 0xf8eb, 0xf8ec, 0xf8ed, 0xf8ee, 0xf8ef, 0xf8f0,
+			0xf8f1, 0xf8f2, 0xf8f3, 0xf8f4, 0xf8f5, 0xf8f6, 0xf8f7, 0xf8f8, 0xf8f9, 0xf8fa,
+			0xf8fb, 0xf8fc, 0xf8fd, 0xf8fe, 0xfe7c, 0xfe7d);
+
+		$pattern = preg_quote(utf8_from_unicode($UTF8_SPECIAL_CHARS), '/');
+		$pattern = '/[\x00-\x19'.$pattern.']/u';
+	}
+
+	return $pattern;
+}
+
+/**
+* Checks a string for whether it contains only word characters. This
+* is logically equivalent to the \w PCRE meta character. Note that
+* this is not a 100% guarantee that the string only contains alpha /
+* numeric characters but just that common non-alphanumeric are not
+* in the string, including ASCII device control characters.
+* @package utf8
+* @subpackage utils
+* @param string to check
+* @return boolean TRUE if the string only contains word characters
+* @see utf8_specials_pattern
+*/
+function utf8_is_word_chars($str)
+{
+	return !(bool) preg_match(utf8_specials_pattern(), $str);
+}
+
+/**
+* Removes special characters (nonalphanumeric) from a UTF-8 string
+*
+* This can be useful as a helper for sanitizing a string for use as
+* something like a file name or a unique identifier. Be warned though
+* it does not handle all possible non-alphanumeric characters and is
+* not intended is some kind of security / injection filter.
+*
+* @package utf8
+* @subpackage utils
+* @author Andreas Gohr <andi@splitbrain.org>
+* @param string $string The UTF8 string to strip of special chars
+* @param string (optional) $repl   Replace special with this string
+* @return string with common non-alphanumeric characters removed
+* @see utf8_specials_pattern
+*/
+function utf8_strip_specials($string, $repl='')
+{
+	return preg_replace(utf8_specials_pattern(), $repl, $string);
+}
diff --git a/include/utf8/utils/unicode.php b/include/utf8/utils/unicode.php
new file mode 100644
index 0000000..f0e86cb
--- /dev/null
+++ b/include/utf8/utils/unicode.php
@@ -0,0 +1,241 @@
+<?php
+
+/**
+* @version $Id: unicode.php,v 1.2 2006/02/26 13:20:44 harryf Exp $
+* Tools for conversion between UTF-8 and unicode
+* The Original Code is Mozilla Communicator client code.
+* The Initial Developer of the Original Code is
+* Netscape Communications Corporation.
+* Portions created by the Initial Developer are Copyright (C) 1998
+* the Initial Developer. All Rights Reserved.
+* Ported to PHP by Henri Sivonen (http://hsivonen.iki.fi)
+* Slight modifications to fit with phputf8 library by Harry Fuecks (hfuecks gmail com)
+* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUTF8ToUnicode.cpp
+* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUnicodeToUTF8.cpp
+* @see http://hsivonen.iki.fi/php-utf8/
+* @package utf8
+* @subpackage unicode
+*/
+
+/**
+* Takes an UTF-8 string and returns an array of ints representing the
+* Unicode characters. Astral planes are supported ie. the ints in the
+* output can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates
+* are not allowed.
+* Returns false if the input string isn't a valid UTF-8 octet sequence
+* and raises a PHP error at level E_USER_WARNING
+* Note: this function has been modified slightly in this library to
+* trigger errors on encountering bad bytes
+* @author <hsivonen@iki.fi>
+* @param string UTF-8 encoded string
+* @return mixed array of unicode code points or FALSE if UTF-8 invalid
+* @see utf8_from_unicode
+* @see http://hsivonen.iki.fi/php-utf8/
+* @package utf8
+* @subpackage unicode
+*/
+function utf8_to_unicode($str)
+{
+	$mState = 0;	// Cached expected number of octets after the current octet
+					// until the beginning of the next UTF8 character sequence
+	$mUcs4  = 0;	// Cached Unicode character
+	$mBytes = 1;	// Cached expected number of octets in the current sequence
+
+	$out = array();
+	$len = strlen($str);
+
+	for($i = 0; $i < $len; $i++)
+	{
+		$in = ord($str[$i]);
+
+		if ($mState == 0)
+		{
+			// When mState is zero we expect either a US-ASCII character or a multi-octet sequence.
+			if (0 == (0x80 & ($in)))
+			{
+				// US-ASCII, pass straight through.
+				$out[] = $in;
+				$mBytes = 1;
+			}
+			else if (0xC0 == (0xE0 & ($in)))
+			{
+				// First octet of 2 octet sequence
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 0x1F) << 6;
+				$mState = 1;
+				$mBytes = 2;
+			}
+			else if (0xE0 == (0xF0 & ($in)))
+			{
+				// First octet of 3 octet sequence
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 0x0F) << 12;
+				$mState = 2;
+				$mBytes = 3;
+			}
+			else if (0xF0 == (0xF8 & ($in)))
+			{
+				// First octet of 4 octet sequence
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 0x07) << 18;
+				$mState = 3;
+				$mBytes = 4;
+			}
+			else if (0xF8 == (0xFC & ($in)))
+			{
+				/* First octet of 5 octet sequence.
+				*
+				* This is illegal because the encoded codepoint must be either
+				* (a) not the shortest form or
+				* (b) outside the Unicode range of 0-0x10FFFF.
+				* Rather than trying to resynchronize, we will carry on until the end
+				* of the sequence and let the later error handling code catch it.
+				*/
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 0x03) << 24;
+				$mState = 4;
+				$mBytes = 5;
+			}
+			else if (0xFC == (0xFE & ($in)))
+			{
+				// First octet of 6 octet sequence, see comments for 5 octet sequence.
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 1) << 30;
+				$mState = 5;
+				$mBytes = 6;
+			}
+			else
+			{
+				// Current octet is neither in the US-ASCII range nor a legal first octet of a multi-octet sequence
+				trigger_error('utf8_to_unicode: Illegal sequence identifier in UTF-8 at byte '.$i, E_USER_WARNING);
+				return false;
+			}
+		}
+		else
+		{
+			// When mState is non-zero, we expect a continuation of the multi-octet sequence
+			if (0x80 == (0xC0 & ($in)))
+			{
+				// Legal continuation.
+				$shift = ($mState - 1) * 6;
+				$tmp = $in;
+				$tmp = ($tmp & 0x0000003F) << $shift;
+				$mUcs4 |= $tmp;
+
+				/**
+				* End of the multi-octet sequence. mUcs4 now contains the final
+				* Unicode codepoint to be output
+				*/
+				if (0 == --$mState)
+				{
+					/*
+					* Check for illegal sequences and codepoints.
+					*/
+					// From Unicode 3.1, non-shortest form is illegal
+					if (((2 == $mBytes) && ($mUcs4 < 0x0080)) || ((3 == $mBytes) && ($mUcs4 < 0x0800)) ||
+					   ((4 == $mBytes) && ($mUcs4 < 0x10000)) || (4 < $mBytes) ||
+					   // From Unicode 3.2, surrogate characters are illegal
+					   (($mUcs4 & 0xFFFFF800) == 0xD800) ||
+					   // Codepoints outside the Unicode range are illegal
+					   ($mUcs4 > 0x10FFFF))
+					{
+						trigger_error('utf8_to_unicode: Illegal sequence or codepoint in UTF-8 at byte '.$i, E_USER_WARNING);
+						return false;
+					}
+
+					// BOM is legal but we don't want to output it
+					if (0xFEFF != $mUcs4)
+						$out[] = $mUcs4;
+
+					// Initialize UTF8 cache
+					$mState = 0;
+					$mUcs4  = 0;
+					$mBytes = 1;
+				}
+
+			}
+			else
+			{
+				/* ((0xC0 & (*in) != 0x80) && (mState != 0))
+				Incomplete multi-octet sequence. */
+				trigger_error('utf8_to_unicode: Incomplete multi-octet sequence in UTF-8 at byte '.$i, E_USER_WARNING);
+				return false;
+			}
+		}
+	}
+
+	return $out;
+}
+
+/**
+* Takes an array of ints representing the Unicode characters and returns
+* a UTF-8 string. Astral planes are supported ie. the ints in the
+* input can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates
+* are not allowed.
+* Returns false if the input array contains ints that represent
+* surrogates or are outside the Unicode range
+* and raises a PHP error at level E_USER_WARNING
+* Note: this function has been modified slightly in this library to use
+* output buffering to concatenate the UTF-8 string (faster) as well as
+* reference the array by it's keys
+* @param array of unicode code points representing a string
+* @return mixed UTF-8 string or FALSE if array contains invalid code points
+* @author <hsivonen@iki.fi>
+* @see utf8_to_unicode
+* @see http://hsivonen.iki.fi/php-utf8/
+* @package utf8
+* @subpackage unicode
+*/
+function utf8_from_unicode($arr)
+{
+	ob_start();
+
+	foreach (array_keys($arr) as $k)
+	{
+		if ( ($arr[$k] >= 0) && ($arr[$k] <= 0x007f) ) // ASCII range (including control chars)
+		{
+			echo chr($arr[$k]);
+		}
+		else if ($arr[$k] <= 0x07ff) //2 byte sequence
+		{
+			echo chr(0xc0 | ($arr[$k] >> 6));
+			echo chr(0x80 | ($arr[$k] & 0x003f));
+		}
+		else if($arr[$k] == 0xFEFF) // Byte order mark (skip)
+		{
+			// Nop -- zap the BOM
+		}
+		else if ($arr[$k] >= 0xD800 && $arr[$k] <= 0xDFFF) // Test for illegal surrogates
+		{
+			// Found a surrogate
+			trigger_error('utf8_from_unicode: Illegal surrogate at index: '.$k.', value: '.$arr[$k], E_USER_WARNING);
+
+			return false;
+		}
+		else if ($arr[$k] <= 0xffff) // 3 byte sequence
+		{
+			echo chr(0xe0 | ($arr[$k] >> 12));
+			echo chr(0x80 | (($arr[$k] >> 6) & 0x003f));
+			echo chr(0x80 | ($arr[$k] & 0x003f));
+		}
+		else if ($arr[$k] <= 0x10ffff) // 4 byte sequence
+		{
+			echo chr(0xf0 | ($arr[$k] >> 18));
+			echo chr(0x80 | (($arr[$k] >> 12) & 0x3f));
+			echo chr(0x80 | (($arr[$k] >> 6) & 0x3f));
+			echo chr(0x80 | ($arr[$k] & 0x3f));
+		}
+		else
+		{
+			trigger_error('utf8_from_unicode: Codepoint out of Unicode range at index: '.$k.', value: '.$arr[$k], E_USER_WARNING);
+
+			// Out of range
+			return false;
+		}
+	}
+
+	$result = ob_get_contents();
+	ob_end_clean();
+
+	return $result;
+}
diff --git a/include/utf8/utils/validation.php b/include/utf8/utils/validation.php
new file mode 100644
index 0000000..90dce8e
--- /dev/null
+++ b/include/utf8/utils/validation.php
@@ -0,0 +1,186 @@
+<?php
+
+/**
+* @version $Id: validation.php,v 1.2 2006/02/26 13:20:44 harryf Exp $
+* Tools for validing a UTF-8 string is well formed.
+* The Original Code is Mozilla Communicator client code.
+* The Initial Developer of the Original Code is
+* Netscape Communications Corporation.
+* Portions created by the Initial Developer are Copyright (C) 1998
+* the Initial Developer. All Rights Reserved.
+* Ported to PHP by Henri Sivonen (http://hsivonen.iki.fi)
+* Slight modifications to fit with phputf8 library by Harry Fuecks (hfuecks gmail com)
+* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUTF8ToUnicode.cpp
+* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUnicodeToUTF8.cpp
+* @see http://hsivonen.iki.fi/php-utf8/
+* @package utf8
+* @subpackage validation
+*/
+
+/**
+* Tests a string as to whether it's valid UTF-8 and supported by the
+* Unicode standard
+* Note: this function has been modified to simple return true or false
+* @author <hsivonen@iki.fi>
+* @param string UTF-8 encoded string
+* @return boolean true if valid
+* @see http://hsivonen.iki.fi/php-utf8/
+* @see utf8_compliant
+* @package utf8
+* @subpackage validation
+*/
+function utf8_is_valid($str)
+{
+	$mState = 0;	// Cached expected number of octets after the current octet
+					// until the beginning of the next UTF8 character sequence
+	$mUcs4  = 0;	// Cached Unicode character
+	$mBytes = 1;	// Cached expected number of octets in the current sequence
+
+	$len = strlen($str);
+
+	for($i = 0; $i < $len; $i++)
+	{
+		$in = ord($str{$i});
+
+		if ( $mState == 0)
+		{
+			// When mState is zero we expect either a US-ASCII character or a multi-octet sequence.
+			if (0 == (0x80 & ($in)))
+			{
+				$mBytes = 1; // US-ASCII, pass straight through
+			}
+			else if (0xC0 == (0xE0 & ($in)))
+			{
+				// First octet of 2 octet sequence
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 0x1F) << 6;
+				$mState = 1;
+				$mBytes = 2;
+			}
+			else if (0xE0 == (0xF0 & ($in)))
+			{
+				// First octet of 3 octet sequence
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 0x0F) << 12;
+				$mState = 2;
+				$mBytes = 3;
+			}
+			else if (0xF0 == (0xF8 & ($in)))
+			{
+				// First octet of 4 octet sequence
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 0x07) << 18;
+				$mState = 3;
+				$mBytes = 4;
+			}
+			else if (0xF8 == (0xFC & ($in)))
+			{
+				/* First octet of 5 octet sequence.
+				*
+				* This is illegal because the encoded codepoint must be either
+				* (a) not the shortest form or
+				* (b) outside the Unicode range of 0-0x10FFFF.
+				* Rather than trying to resynchronize, we will carry on until the end
+				* of the sequence and let the later error handling code catch it.
+				*/
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 0x03) << 24;
+				$mState = 4;
+				$mBytes = 5;
+			}
+			else if (0xFC == (0xFE & ($in)))
+			{
+				// First octet of 6 octet sequence, see comments for 5 octet sequence.
+				$mUcs4 = ($in);
+				$mUcs4 = ($mUcs4 & 1) << 30;
+				$mState = 5;
+				$mBytes = 6;
+			}
+			else
+			{
+				// Current octet is neither in the US-ASCII range nor a legal first octet of a multi-octet sequence.
+				return false;
+			}
+		}
+		else
+		{
+			// When mState is non-zero, we expect a continuation of the multi-octet sequence
+			if (0x80 == (0xC0 & ($in)))
+			{
+				// Legal continuation.
+				$shift = ($mState - 1) * 6;
+				$tmp = $in;
+				$tmp = ($tmp & 0x0000003F) << $shift;
+				$mUcs4 |= $tmp;
+
+				/**
+				* End of the multi-octet sequence. mUcs4 now contains the final
+				* Unicode codepoint to be output
+				*/
+				if (0 == --$mState)
+				{
+					/*
+					* Check for illegal sequences and codepoints.
+					*/
+					// From Unicode 3.1, non-shortest form is illegal
+					if (((2 == $mBytes) && ($mUcs4 < 0x0080)) || ((3 == $mBytes) && ($mUcs4 < 0x0800)) ||
+					((4 == $mBytes) && ($mUcs4 < 0x10000)) || (4 < $mBytes) ||
+					// From Unicode 3.2, surrogate characters are illegal
+					(($mUcs4 & 0xFFFFF800) == 0xD800) ||
+					// Codepoints outside the Unicode range are illegal
+					($mUcs4 > 0x10FFFF))
+					{
+						return FALSE;
+					}
+
+					// Initialize UTF8 cache
+					$mState = 0;
+					$mUcs4  = 0;
+					$mBytes = 1;
+				}
+			}
+			else
+			{
+				/**
+				*((0xC0 & (*in) != 0x80) && (mState != 0))
+				* Incomplete multi-octet sequence.
+				*/
+
+				return false;
+			}
+		}
+	}
+
+	return true;
+}
+
+/**
+* Tests whether a string complies as UTF-8. This will be much
+* faster than utf8_is_valid, but will pass five and six octet
+* UTF-8 sequences, which are not supported by Unicode and
+* so cannot be displayed correctly in a browser. In other words
+* it is not as strict as utf8_is_valid but it's faster. If you use
+* is to validate user input, you place yourself at the risk that
+* attackers will be able to inject 5 and 6 byte sequences (which
+* may or may not be a significant risk, depending on what you are
+* are doing)
+* Note: Does not pass five and six octet UTF-8 sequences anymore in
+*       in the unit tests.
+* @see utf8_is_valid
+* @see http://www.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805
+* @param string UTF-8 string to check
+* @return boolean TRUE if string is valid UTF-8
+* @package utf8
+* @subpackage validation
+*/
+function utf8_compliant($str)
+{
+	if (strlen($str) == 0)
+		return true;
+
+	// If even just the first character can be matched, when the /u
+	// modifier is used, then it's valid UTF-8. If the UTF-8 is somehow
+	// invalid, nothing at all will match, even if the string contains
+	// some valid sequences
+	return (preg_match('/^.{1}/us', $str, $ar) == 1);
+}
-- 
cgit v1.2.3-54-g00ecf