<?php
 class vB_RelatedText_Vocabulary { use vB_Trait_NoSerialize; private vB_Utility_String $stringutil; private int $minlength; private int $maxlength; private array $inverseVocabulary = []; private array $doccounts = []; private int $totalDocs = 0; public static function createFromFile(string $filename, vB_Utility_String $stringutil) : self { $vocab = new self($stringutil, 0, 0); $vocab->readVocabulary($filename); return $vocab; } public function __construct(vB_Utility_String $stringutil, int $minlength, int $maxlength) { $this->stringutil = $stringutil; $this->minlength = $minlength; $this->maxlength = $maxlength; } public function getCount() : int { return count($this->inverseVocabulary); } public function getVocabulary() : array { return array_flip($this->inverseVocabulary); } public function wordToOrdinal(string $word) : ?int { return $this->inverseVocabulary[$word] ?? null; } public function ordinalToWord(int $ordinal) : ?string { $word = array_search($ordinal, $this->inverseVocabulary); if ($word !== false) { return $word; } return null; } public function getDocCounts() : array { return $this->doccounts; } public function getTotalDocCount() : int { return $this->totalDocs; } public function processText(string $text) : array { $this->totalDocs++; $tokens = $this->tokenize($text); $counts = []; foreach ($tokens AS $token) { $tokenlen = $this->stringutil->strlen($token); if ($tokenlen >= $this->minlength AND $tokenlen <= $this->maxlength) { $this->inverseVocabulary[$token] ??= count($this->inverseVocabulary); $ordinal = $this->inverseVocabulary[$token]; if (!isset($counts[$ordinal])) { $counts[$ordinal] = 0; $this->doccounts[$ordinal] = ($this->doccounts[$ordinal] ?? 0) + 1; } $counts[$ordinal]++; } } ksort($counts); return $counts; } public function filterTermsByFrequency(int $minDocCount, float $maxDocRatio) : void { $vocab = array_flip($this->inverseVocabulary); unset($this->inverseVocabulary); foreach ($this->doccounts AS $ordinal => $count) { if ($count < $minDocCount OR (($count / $this->totalDocs) > $maxDocRatio)) { unset($this->doccounts[$ordinal]); unset($vocab[$ordinal]); } } $this->inverseVocabulary = array_flip($vocab); } public function filterRemovedTerms(array &$data) : void { foreach ($data AS &$record) { foreach ($record AS $ordinal => $count) { if (!isset($this->doccounts[$ordinal])) { unset($record[$ordinal]); } } } } public function writeVocabulary(string $filename) : void { $persist = new vB_RelatedText_Persist($filename, false); $persist->writeInt($this->minlength); $persist->writeInt($this->maxlength); $persist->writeStringKeyIntArray($this->inverseVocabulary); $persist->writeInt($this->totalDocs); $persist->writeIntArray($this->doccounts); } private function readVocabulary(string $filename) : void { $persist = new vB_RelatedText_Persist($filename, true); $this->minlength = $persist->readInt(); $this->maxlength = $persist->readInt(); $this->inverseVocabulary = $persist->readStringKeyIntArray(); $this->totalDocs = $persist->readInt(); $this->doccounts = $persist->readIntArray(); } private function tokenize(string $text) : array { $matches = []; preg_match_all("/[\w'-]+/u", $text, $matches); return $matches[0]; } } 