TranslationWordList.php

Go to the documentation of this file.

<?php
/***************************************************************************
 *   Copyright (C) 2005-2020 by Ferenc Veres                                                       *
 *   lion@netngine.hu                                                                                                      *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 3 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>. *
 ***************************************************************************/
include_once('TranslationWord.php');

class TranslationWordList implements Iterator
{
        private $allWords;              
        private $fakeLength;    
        private $DB;                    
        private $Tables;                
        private $currIndex;             
        public static $WordsCache; 
        function __construct($text = "")
        {
                // Init DB references
                $this->DB = GetDbConn();
                $this->Tables = GetTables();

                $this->currIndex = 0;
                $this->fakeLength = 0;
                unset($this->allWords);

                // If text passed, parse it and fill the words arrays (also adds to WordIndex if necessary)
                if($text != "")
                {
                        $this->SplitToWords($text);
                }
        }

        /**** Iterator interface ****/

        public function rewind()
        {
                $this->currIndex = 0;
        }
        public function hasMore()
        {
                return count($this->allWords) > $this->currIndex;
        }
        public function key()
        {
                return $this->currIndex;
        }
        public function current()
        {
                return $this->allWords[$this->currIndex];
        }
        public function next()
        {
                $this->currIndex++;
        }
        public function valid()
        {
        if(count($this->allWords) > $this->currIndex)
                {
                        return true;
                }
                return false;
        }

        /* Static initialization methods */

        public static function InitWordsCache()
        {
                $Tables = GetTables();
                $DB = GetDbConn();
                
                // Fetch already indexed words and their ID
                $rs = $DB->Execute("SELECT Word, ID FROM ".$Tables["WordIndex"]);
        
                if(!$rs)
                {
                        die("DB Error:". $this->DB->ErrorMsg());
                }
        
                while($oldWord = $rs->FetchRow())
                {
                        // Already existing words get an ID now
                        TranslationWordList::$WordsCache[$oldWord["Word"]] = $oldWord["ID"];
                }
        }

        /**** Add words and properties ****/
        
        function AddWord($word, $count = 1)
        {
                // Get a reference to this word in allWords if already added
                $w =& $this->FindByName($word);
                if($w == null)
                {
                        // First occurence(s) in this sentence, create with emtpy ID
                        $w = new TranslationWord();
                        $w->SetWord($word);
                        $w->SetCount($count);
                        $this->allWords[] = $w;
                }
                else
                {
                        // Not first occurence(s), count it
                        $w->SetCount($w->GetCount() + $count);
                }
        }

        function AddID($id, $count = 1)
        {
                // Get a reference to this word in allWords if already added
                $w =& $this->FindByID($id);
                if($w == null)
                {
                        // First occurence(s) in this sentence, create with emtpy Name
                        $w = new TranslationWord();
                        $w->SetID($id);
                        $w->SetCount($count);
                        $this->allWords[] = $w;
                }
                else
                {
                        // Not first occurence(s), count it
                        $w->SetCount($w->GetCount() + $count);
                }
        }


        function SetIDByName($word, $id)
        {
                $w =& $this->FindByName($word);
                if($w == null)
                {
                        die("WORDS: SetIDByName to a not existing word: $word, $id.\n");
                }

                $w->SetID($id);
        }
        
        function SetFakeLength($length)
        {
                $this->fakeLength = $length;
        }

        private function SplitToWords($text)
        {
                // Clear previous items
                unset($this->allWords);
                $this->SetFakeLength(0);
                
                // Remove linebreaks
                $text = str_replace("\n", " ", $text);
                $text = str_replace("\r", " ", $text);
                
                // Remove some HTML emphasis from the words
                $text = str_replace("<strong>", "", $text);
                $text = str_replace("</strong>", "", $text);
                $text = str_replace("<b>", "", $text);
                $text = str_replace("</b>", "", $text);
                $text = str_replace("<i>", "", $text);
                $text = str_replace("</i>", "", $text);
                $text = str_replace("<em>", "", $text);
                $text = str_replace("</em>", "", $text);
                $text = str_replace("&nbsp;", " ", $text);

                // Split to words
                $wordList = preg_split('/[- \.,()!\?\/\\<>_:;=|\\*]/', $text, -1, PREG_SPLIT_NO_EMPTY);
                
                // Filter non-words
                foreach($wordList as $word)
                {
                        // Remove trash remained with split
                        $word = trim($word, " \t\n\r:[]#'\"$+-_<>/");

                        // Nonempty, nonnumbers
                        if($word != "" && !is_numeric($word))
                        {
                                $this->AddWord(strtolower($word));
                        }
                }

                // Add to word index database and get indexes for already indexed
                if($this->SumCount() != 0)
                {
                        $this->AddToWordIndex();
                }
        }

        /**** Get words and properties ****/

        function &FindByName($word)
        {
                if(empty($this->allWords))
                {
                        $null = null;
                        return $null;
                }
                
                foreach($this->allWords as $w)
                {
                        if($w->GetWord() == $word)
                        {
                                return $w;
                        }
                }

                $null = null;
                return $null;
        }

        function &FindByID($id)
        {
                if(empty($this->allWords))
                {
                        $null = null;
                        return $null;
                }
                
                foreach($this->allWords as $w)
                {
                        if($w->GetID() == $id)
                        {
                                return $w;
                        }
                }

                $null = null;
                return $null;
        }

        
        function SumCount()
        {
                // Some sentences are not completely loaded but can fake their
                // original length
                if($this->fakeLength <> 0)
                {
                        return $this->fakeLength;
                }

                // Not loaded sentences are 0 long
                if(empty($this->allWords))
                {
                        return 0;
                }

                // Summarize all words occurences otherwise
                $count = 0;
                foreach($this->allWords as $w)
                {
                        $count += $w->GetCount();
                }

                return $count;
        }

        /**** Compare ****/

        function CompareByID($otherWords)
        {
                // Max points is the sum of all words from both sentences
                $max = $this->SumCount() + $otherWords->SumCount();
                
                $points = 0;
                
                // Loop on all our own words
                foreach($this->allWords as $w)
                {
                        //This word exists in the other sentence too?
                        $w2 = $otherWords->FindByID($w->GetID());
                        if($w2 != null)
                        {
                                // Yes. common word. (each common word gets 2 points)
                                
                                // Same number of occurences?
                                if($w2->GetCount() == $w->GetCount())
                                {
                                        $points += $w->GetCount() * 2;
                                }
                                else
                                {
                                        $points += abs($w2->GetCount() - $w->GetCount()) * 2;
                                }
                        }
                }

                // Return match percentage

                return floor((100/$max) * $points);
        }

        /**** Database stuff ****/

        private function AddToWordIndex()
        {
                // Get references to cached data (also inits the cache if necessary)
                
                // Add all new words to the index ($w is reference!)
                foreach($this->allWords as &$w)
                {
                        // Not yet existing word?
                        if(!empty(TranslationWordList::$WordsCache[$w->GetWord()]))
                        {
                                // Have word
                                $this->SetIDByName($w->GetWord(), TranslationWordList::$WordsCache[$w->GetWord()]);
                        }
                        else
                        {
                                // Add new word to the index and get it's ID
                                $ok = $this->DB->Execute("INSERT INTO ".$this->Tables["WordIndex"]." (Word) VALUES ('".fixstr($w->GetWord())."')");

                                if(!$ok)
                                {
                                        die("DB Error:". $this->DB->ErrorMsg()." Line: ".__LINE__);
                                }
                                if(!$this->DB->Insert_ID())
                                {
                                        die("DB Error: Insert_ID not supported. Line: ".__LINE__);
                                }
                                
                                // Store the new ID
                                $w->SetID($this->DB->Insert_ID());
                                TranslationWordList::$WordsCache[$w->GetWord()] = $w->GetID();
                        }
                }

                // By now all Words have a name, ID and count for this Sentence
        }

}


?>