Mozilla Universal Charset Detect on C/C++/PHP

Mozilla (C++)        -> mxr.mozilla.org/mozilla-central/source/intl/chardet/src/ 
Nchardet (C#)        -> conceptdevelopment.net/Localization/NCharDet/ 
Jchardet (Java)      -> jchardet.sourceforge.net/ 
chardet (Python)     -> chardet.feedparser.org/ 
chardet (ryby)       -> rubyforge.org/projects/chardet/ 
Encode-Detect (perl) -> search.cpan.org/~jgmyers/Encode-Detect-1.01/Detector.pm 

1. Latin2(Hungarian), Win1250(Hungarian), TIS620(ThaiModel)
   를 detecting 하지 못하는 문제 해결
2. C wraping function 제공 (C/C++ support)
3. libtool 을 이용해서 so/a library 제공
4. chardet-config 명령을 이용한 빌드 환경 제공

/*
 * sample code with libchardet
 * author: JoungKyun.Kim <http://oops.org>
 * $Id: sample.c,v 1.1.1.1 2009/02/21 20:08:02 oops Exp $
 */
 
#include <chardet.h>
 
int main (void) {
    Detect * d;
    DetectObj *obj;
    int i;
 
    char *str[] = {
        "안녕",
        "안녕하세요",
        "안녕하세요 정말?",
        "그래 이정도면 판단 될까?",
        "좀더 길게 적어 볼까 얼마나 길게 해야!",
        "그래 그래 좀 더 길게 적어 보자 더 길게 적어 보야야 겠지...",
        "12345 abcde"
    };
 
    short arrayNum;
    arrayNum = sizeof (str) / sizeof (str[0]);
 
    if ( (d = detect_init ()) == NULL ) {
        fprintf (stderr, "chardet handle initialize failed\n");
        return CHARDET_MEM_ALLOCATED_FAIL;
    }
 
    for ( i=0; i<arrayNum; i++ ) {
        detect_reset (&d);
 
        obj = detect_obj_init ();
        if ( obj == NULL ) {
            fprintf (stderr, "On attemped \"%s\", memory allocation failed\n", str[i]);
            continue;
        }
 
        if ( detect_handledata (&d, str[i], &obj) == CHARDET_OUT_OF_MEMORY ) {
            fprintf (stderr, "On handle processing, occured out of memory\n");
            return CHARDET_OUT_OF_MEMORY;
        }
        printf ("## %s : %s : %f : %d\n", str[i], obj->encoding, obj->confidence, obj->status);
        detect_obj_free (&obj);
    }
 
    detect_destroy (&d);
 
    return 0;
}

/*
 * sample code with libchardet
 * author: JoungKyun.Kim <http://oops.org>
 * $Id: sample1.c,v 1.1.1.1 2009/02/21 20:08:02 oops Exp $
 */
#include <chardet.h>
//#include "../src/chardet.h"
 
int main (void) {
    DetectObj *obj;
    int i;
 
    char *str[] = {
        "안녕",
        "안녕하세요",
        "안녕하세요 정말?",
        "그래 이정도면 판단 될까?",
        "좀더 길게 적어 볼까 얼마나 길게 해야!",
        "그래 그래 좀 더 길게 적어 보자 더 길게 적어 보야야 겠지...",
        "12345 abcde"
    };
 
    short arrayNum;
    arrayNum = sizeof (str) / sizeof (str[0]);
 
    for ( i=0; i<arrayNum; i++ ) {
        obj = detect_obj_init ();
        if ( obj == NULL ) {
            fprintf (stderr, "On attemped \"%s\", memory allocation failed\n", str[i]);
            continue;
        }
 
        if ( detect (str[i], &obj) == CHARDET_OUT_OF_MEMORY ) {
            fprintf (stderr, "On handle processing, occured out of memory\n");
            return CHARDET_OUT_OF_MEMORY;
        }
        printf ("## %s : %s : %f : %d\n", str[i], obj->encoding, obj->confidence, obj->status);
        detect_obj_free (&obj);
    }
 
    return 0;
}

<?
if ( ! extension_loaded ('chardet') )
    dl ('chardet.so');
 
$strings = array (
    '안녕하세요 abc는 영어고요, 가나다는 한글 입니다.',
    '안녕',
    '안녕하세요',
    '조금더 길게 적어 봅니다. 어느 정도가 필요할까요? 오호라.. 점점 길어지네',
);
 
$fp = chardet_open ();
 
$i=0;
foreach ( $strings as $s ) {
    #
    # proto object chardet_detect (stream handle, string[, mode])
    #   database handle : return value of chardet_open () API
    #   string          : strings for character set detecting
    #   mode            : optional
    #                     if support CHARDTE_MOZ, this value is
    #                     default, and don't support CHARDET_MOZ,
    #                     CHARDET_ICU is default.
    #
    #                     CHARDET_MOZ : libchardet library result
    #                     CHARDET_ICU : icu library result
    #                     CHARDET_PY  : python-chardet result
    #
    #                     if each CHARDET_(MOZ|ICU|PY) value is -1,
    #                     it means don't support each mode.
    #
    #  return value type : object -> encoding    : detecting charset name
    #                                langs       : charset language name
    #                                              Don't support on CHARDET_MOZ and CAHRDET_PY mode
    #                                confidence  : detecting confidence
    #                                status      : error code (0 is not error)
    #
    if ( CHARDET_MOZ != -1 )
        $moz = chardet_detect ($fp, $s);
    if ( CHARDET_ICU != -1 )
        $icu = chardet_detect ($fp, $s, CHARDET_ICU);
    if ( CHARDET_PY != -1 )
        $py = chardet_detect ($fp, $s, CHARDET_PY);
 
    echo "$s\n";
    if ( CHARDET_MOZ != -1 )
        printf ("MOZ : Encoding -> %-12s, Confidence -> %3d, Status -> %d\n",
                $moz->encoding, $moz->confidence, $moz->status);
    if ( CHARDET_ICU != -1 )
        printf ("ICU : Encoding -> %-12s, Confidence -> %3d, Status -> %d\n",
                $icu->encoding, $icu->confidence, $icu->status);
    if ( CHARDET_PY != -1 )
        printf ("PY  : Encoding -> %-12s, Confidence -> %3d, Status -> %d\n",
                $py->encoding, $py->confidence, $py->status);
    echo "\n";
 
    $i++;
}
 
chardet_close ($fp);
?>