|
Packit |
67f6e7 |
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
|
|
Packit |
67f6e7 |
/* ***** BEGIN LICENSE BLOCK *****
|
|
Packit |
67f6e7 |
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
|
|
Packit |
67f6e7 |
*
|
|
Packit |
67f6e7 |
* The contents of this file are subject to the Mozilla Public License Version
|
|
Packit |
67f6e7 |
* 1.1 (the "License"); you may not use this file except in compliance with
|
|
Packit |
67f6e7 |
* the License. You may obtain a copy of the License at
|
|
Packit |
67f6e7 |
* http://www.mozilla.org/MPL/
|
|
Packit |
67f6e7 |
*
|
|
Packit |
67f6e7 |
* Software distributed under the License is distributed on an "AS IS" basis,
|
|
Packit |
67f6e7 |
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
|
|
Packit |
67f6e7 |
* for the specific language governing rights and limitations under the
|
|
Packit |
67f6e7 |
* License.
|
|
Packit |
67f6e7 |
*
|
|
Packit |
67f6e7 |
* The Original Code is Mozilla Universal charset detector code.
|
|
Packit |
67f6e7 |
*
|
|
Packit |
67f6e7 |
* The Initial Developer of the Original Code is
|
|
Packit |
67f6e7 |
* Netscape Communications Corporation.
|
|
Packit |
67f6e7 |
* Portions created by the Initial Developer are Copyright (C) 2001
|
|
Packit |
67f6e7 |
* the Initial Developer. All Rights Reserved.
|
|
Packit |
67f6e7 |
*
|
|
Packit |
67f6e7 |
* Contributor(s):
|
|
Packit |
67f6e7 |
* Shy Shalom <shooshX@gmail.com>
|
|
Packit |
67f6e7 |
* Proofpoint, Inc.
|
|
Packit |
67f6e7 |
*
|
|
Packit |
67f6e7 |
* Alternatively, the contents of this file may be used under the terms of
|
|
Packit |
67f6e7 |
* either the GNU General Public License Version 2 or later (the "GPL"), or
|
|
Packit |
67f6e7 |
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
|
|
Packit |
67f6e7 |
* in which case the provisions of the GPL or the LGPL are applicable instead
|
|
Packit |
67f6e7 |
* of those above. If you wish to allow use of your version of this file only
|
|
Packit |
67f6e7 |
* under the terms of either the GPL or the LGPL, and not to allow others to
|
|
Packit |
67f6e7 |
* use your version of this file under the terms of the MPL, indicate your
|
|
Packit |
67f6e7 |
* decision by deleting the provisions above and replace them with the notice
|
|
Packit |
67f6e7 |
* and other provisions required by the GPL or the LGPL. If you do not delete
|
|
Packit |
67f6e7 |
* the provisions above, a recipient may use your version of this file under
|
|
Packit |
67f6e7 |
* the terms of any one of the MPL, the GPL or the LGPL.
|
|
Packit |
67f6e7 |
*
|
|
Packit |
67f6e7 |
* ***** END LICENSE BLOCK ***** */
|
|
Packit |
67f6e7 |
#include <stdio.h>
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
#include "nsMBCSGroupProber.h"
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
#if defined(DEBUG_chardet) || defined(DEBUG_jgmyers)
|
|
Packit |
67f6e7 |
const char *ProberName[] =
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
"UTF8",
|
|
Packit |
67f6e7 |
"SJIS",
|
|
Packit |
67f6e7 |
"EUCJP",
|
|
Packit |
67f6e7 |
"GB18030",
|
|
Packit |
67f6e7 |
"EUCKR",
|
|
Packit |
67f6e7 |
"Big5",
|
|
Packit |
67f6e7 |
"EUCTW",
|
|
Packit |
67f6e7 |
};
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
#endif
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
nsMBCSGroupProber::nsMBCSGroupProber()
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
mProbers[0] = new nsUTF8Prober();
|
|
Packit |
67f6e7 |
mProbers[1] = new nsSJISProber();
|
|
Packit |
67f6e7 |
mProbers[2] = new nsEUCJPProber();
|
|
Packit |
67f6e7 |
mProbers[3] = new nsGB18030Prober();
|
|
Packit |
67f6e7 |
mProbers[4] = new nsEUCKRProber();
|
|
Packit |
67f6e7 |
mProbers[5] = new nsBig5Prober();
|
|
Packit |
67f6e7 |
mProbers[6] = new nsEUCTWProber();
|
|
Packit |
67f6e7 |
Reset();
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
nsMBCSGroupProber::~nsMBCSGroupProber()
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
for (PRUint32 i = 0; i < NUM_OF_PROBERS; i++)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
delete mProbers[i];
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
const char* nsMBCSGroupProber::GetCharSetName()
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
if (mBestGuess == -1)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
GetConfidence();
|
|
Packit |
67f6e7 |
if (mBestGuess == -1)
|
|
Packit |
67f6e7 |
mBestGuess = 0;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
return mProbers[mBestGuess]->GetCharSetName();
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
void nsMBCSGroupProber::Reset(void)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
mActiveNum = 0;
|
|
Packit |
67f6e7 |
for (PRUint32 i = 0; i < NUM_OF_PROBERS; i++)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
if (mProbers[i])
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
mProbers[i]->Reset();
|
|
Packit |
67f6e7 |
mIsActive[i] = PR_TRUE;
|
|
Packit |
67f6e7 |
++mActiveNum;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
else
|
|
Packit |
67f6e7 |
mIsActive[i] = PR_FALSE;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
mBestGuess = -1;
|
|
Packit |
67f6e7 |
mState = eDetecting;
|
|
Packit |
67f6e7 |
mKeepNext = 0;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
nsProbingState nsMBCSGroupProber::HandleData(const char* aBuf, PRUint32 aLen)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
nsProbingState st;
|
|
Packit |
67f6e7 |
PRUint32 start = 0;
|
|
Packit |
67f6e7 |
PRUint32 keepNext = mKeepNext;
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
//do filtering to reduce load to probers
|
|
Packit |
67f6e7 |
for (PRUint32 pos = 0; pos < aLen; ++pos)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
if (aBuf[pos] & 0x80)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
if (!keepNext)
|
|
Packit |
67f6e7 |
start = pos;
|
|
Packit |
67f6e7 |
keepNext = 2;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
else if (keepNext)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
if (--keepNext == 0)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
for (PRUint32 i = 0; i < NUM_OF_PROBERS; i++)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
if (!mIsActive[i])
|
|
Packit |
67f6e7 |
continue;
|
|
Packit |
67f6e7 |
st = mProbers[i]->HandleData(aBuf + start, pos + 1 - start);
|
|
Packit |
67f6e7 |
if (st == eFoundIt)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
mBestGuess = i;
|
|
Packit |
67f6e7 |
mState = eFoundIt;
|
|
Packit |
67f6e7 |
return mState;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
else if (st == eNotMe)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
mIsActive[i] = PR_FALSE;
|
|
Packit |
67f6e7 |
mActiveNum--;
|
|
Packit |
67f6e7 |
if (mActiveNum <= 0)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
mState = eNotMe;
|
|
Packit |
67f6e7 |
return mState;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
if (keepNext) {
|
|
Packit |
67f6e7 |
for (PRUint32 i = 0; i < NUM_OF_PROBERS; i++)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
if (!mIsActive[i])
|
|
Packit |
67f6e7 |
continue;
|
|
Packit |
67f6e7 |
st = mProbers[i]->HandleData(aBuf + start, aLen + 1 - start);
|
|
Packit |
67f6e7 |
if (st == eFoundIt)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
mBestGuess = i;
|
|
Packit |
67f6e7 |
mState = eFoundIt;
|
|
Packit |
67f6e7 |
return mState;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
else if (st == eNotMe)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
mIsActive[i] = PR_FALSE;
|
|
Packit |
67f6e7 |
mActiveNum--;
|
|
Packit |
67f6e7 |
if (mActiveNum <= 0)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
mState = eNotMe;
|
|
Packit |
67f6e7 |
return mState;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
mKeepNext = keepNext;
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
return mState;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
float nsMBCSGroupProber::GetConfidence(void)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
PRUint32 i;
|
|
Packit |
67f6e7 |
float bestConf = 0.0, cf;
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
switch (mState)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
case eFoundIt:
|
|
Packit |
67f6e7 |
return (float)0.99;
|
|
Packit |
67f6e7 |
case eNotMe:
|
|
Packit |
67f6e7 |
return (float)0.01;
|
|
Packit |
67f6e7 |
default:
|
|
Packit |
67f6e7 |
for (i = 0; i < NUM_OF_PROBERS; i++)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
if (!mIsActive[i])
|
|
Packit |
67f6e7 |
continue;
|
|
Packit |
67f6e7 |
cf = mProbers[i]->GetConfidence();
|
|
Packit |
67f6e7 |
if (bestConf < cf)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
bestConf = cf;
|
|
Packit |
67f6e7 |
mBestGuess = i;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
return bestConf;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
#ifdef DEBUG_chardet
|
|
Packit |
67f6e7 |
void nsMBCSGroupProber::DumpStatus()
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
PRUint32 i;
|
|
Packit |
67f6e7 |
float cf;
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
GetConfidence();
|
|
Packit |
67f6e7 |
for (i = 0; i < NUM_OF_PROBERS; i++)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
if (!mIsActive[i])
|
|
Packit |
67f6e7 |
printf(" MBCS inactive: [%s] (confidence is too low).\r\n", ProberName[i]);
|
|
Packit |
67f6e7 |
else
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
cf = mProbers[i]->GetConfidence();
|
|
Packit |
67f6e7 |
printf(" MBCS %1.3f: [%s]\r\n", cf, ProberName[i]);
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
#endif
|
|
Packit |
67f6e7 |
|
|
Packit |
67f6e7 |
#ifdef DEBUG_jgmyers
|
|
Packit |
67f6e7 |
void nsMBCSGroupProber::GetDetectorState(nsUniversalDetector::DetectorState (&states)[nsUniversalDetector::NumDetectors], PRUint32 &offset)
|
|
Packit |
67f6e7 |
{
|
|
Packit |
67f6e7 |
for (PRUint32 i = 0; i < NUM_OF_PROBERS; ++i) {
|
|
Packit |
67f6e7 |
states[offset].name = ProberName[i];
|
|
Packit |
67f6e7 |
states[offset].isActive = mIsActive[i];
|
|
Packit |
67f6e7 |
states[offset].confidence = mIsActive[i] ? mProbers[i]->GetConfidence() : 0.0;
|
|
Packit |
67f6e7 |
++offset;
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
}
|
|
Packit |
67f6e7 |
#endif /* DEBUG_jgmyers */
|