Source for file lucene-defs.php

Documentation is available at lucene-defs.php

  1. <?php
  2. /* ******************************************************************** */
  3. /* CATALYST PHP Source Code */
  4. /* -------------------------------------------------------------------- */
  5. /* This program is free software; you can redistribute it and/or modify */
  6. /* it under the terms of the GNU General Public License as published by */
  7. /* the Free Software Foundation; either version 2 of the License, or */
  8. /* (at your option) any later version. */
  9. /* */
  10. /* This program is distributed in the hope that it will be useful, */
  11. /* but WITHOUT ANY WARRANTY; without even the implied warranty of */
  12. /* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */
  13. /* GNU General Public License for more details. */
  14. /* */
  15. /* You should have received a copy of the GNU General Public License */
  16. /* along with this program; if not, write to: */
  17. /* The Free Software Foundation, Inc., 59 Temple Place, Suite 330, */
  18. /* Boston, MA 02111-1307 USA */
  19. /* -------------------------------------------------------------------- */
  20. /* */
  21. /* Filename: lucene-defs.php */
  22. /* Author: Paul Waite */
  23. /* Description: NB: This module is a variant of the original lucene */
  24. /* module which processed fields line-by-line. This module */
  25. /* implements the XML interface to Lucene. */
  26. /* */
  27. /* Definitions for interfacing to the LUCENE search */
  28. /* engine system. LUCENE is a system which is optimised */
  29. /* for indexing and searching in a generic way. It is */
  30. /* implemented as a server accessible via a port over TCP. */
  31. /* This module understands the protocol that this server */
  32. /* uses to implement indexing and search queries. */
  33. /* */
  34. /* ******************************************************************** */
  35. /** @package search */
  36. include_once("search-defs.php");
  37. /** Stopwatch microtimer */
  38. ("timer-defs.php");
  39. /** XML classes */
  40. ("xml-defs.php");
  41.  
  42. // ----------------------------------------------------------------------
  43. /** Do not wait on socket receive, return immediately */
  44. ("SOCK_NO_WAIT", 0);
  45. /** Wait on socket forever (well, 24hrs is that, more or less) */
  46. ("SOCK_FOREVER", 86400);
  47. /** Times to retry timed-out socket sends/receives */
  48. ("SOCK_RETRIES", 3);
  49. /** Used to indicate that a field should be indexed by Lucene */
  50. ("INDEXED", true);
  51. /** Used to indicate that a field should NOT be indexed by Lucene */
  52. ("NOT_INDEXED", false);
  53. /** Used to indicate that a field should be stored by Lucene */
  54. ("STORED", true);
  55. /** Used to indicate that a field should NOT be stored by Lucene */
  56. ("NOT_STORED", false);
  57. /** The name of the field Lucene should assume if none specified */
  58. ("DEFAULT_FIELD", "Text");
  59. /** Default type of field: 'Text', 'Date', 'Id' */
  60. ("DEFAULT_FIELDTYPE", "Text");
  61. /** Mode of index ID generation is by incrementing integer */
  62. ("ID_FROM_INC", 0);
  63. /** Mode of index ID generation is by filename stripped of path and extension */
  64. ("ID_FROM_NAME", 1);
  65. /** Mode of index ID generation is by full filename (incl. extension) */
  66. ("ID_FROM_FILENAME", 2);
  67. /** Mode of index ID generation is by full path to file */
  68. ("ID_FROM_PATH", 3);
  69. /** Indicates index fields come from meta tag extraction */
  70. ("META_TAG_FIELDS", true);
  71.  
  72. // ----------------------------------------------------------------------
  73. /**
  74. * The lucene connection class
  75. * This class inherits the functionality of the 'search' class since mostly
  76. * that is what we will be connecting to Lucene for. The Indexing and
  77. * Control descendants can just ignore this inherited basic searching
  78. * functionality.
  79. * This class knows how to connect to a Lucene server and send and
  80. * receive messages to/from it. Child classes which need to talk to this
  81. * server to do indexing or querying should inherit this class.
  82. * @package search
  83. */
  84. class lucene_connection extends search {
  85. // Public
  86. /** HOST running the Lucene query server */
  87.  
  88. var $host = "";
  89. /** PORT that the server is listening on */
  90.  
  91. var $port = "";
  92. /** Timeout for send in seconds */
  93.  
  94. var $timeoutsecs = 10;
  95.  
  96. // Private
  97. /** Whether Lucene is enabled..
  98. @access private */
  99. var $enabled = true;
  100. /** The message waiting to be sent
  101. @access private */
  102. var $message = "";
  103. /** Raw response content we receive back from the Lucene server
  104. @access private */
  105. var $responsebuf = "";
  106. /** Socket file pointer
  107. @access private */
  108. var $sockfp = false;
  109. /** True if we are connected to socket
  110. @access private */
  111. var $connected = false;
  112. /** An execution timer
  113. @access private */
  114. var $timer;
  115. // .....................................................................
  116. /** Constructor - Lucene connection
  117. * @param string $host Hostname or IP of Lucene server
  118. * @param string $port Port of Lucene server
  119. * @param integer $timeoutsecs Seconds to timeout the connection
  120. */
  121. function lucene_connection($host="", $port="", $timeoutsecs="") {
  122. debugbr("Lucene connection: using XML interface v1.0");
  123. if ($host != "") {
  124. $this->connect($host, $port, $timeoutsecs);
  125. }
  126. $this->timer = new microtimer();
  127. } // lucene_connection
  128. // .....................................................................
  129. /**
  130. * Connect to the Lucene server. Optionally over-ride various settings
  131. * which were set in the constructor. Normally this method is only
  132. * called internally, in response to a request to send a message to
  133. * the Luceneserver.
  134. * @access private
  135. * @param string $host Hostname or IP of Lucene server
  136. * @param string $port Port of Lucene server
  137. * @param integer $timeoutsecs Seconds to timeout the connection
  138. */
  139. function connect($host="", $port="", $timeoutsecs="") {
  140. // Override host and port if given..
  141. if ($host != "") $this->host = $host;
  142. if ($port != "") $this->port = $port;
  143.  
  144. // Utilise the Axyl configuration settings, if available..
  145. if (class_exists("configuration")) {
  146. $config = new configuration("sys_control");
  147. // This controls whether we have Lucene capability or not..
  148. if ($config->field_exists("Lucene Site Indexing")) {
  149. $this->enabled = $config->value("Lucene Site Indexing");
  150. }
  151. // Only set host & port if they have not been given yet..
  152. if ($this->host == "") {
  153. $this->host = $config->value("Lucene Host");
  154. $this->port = $config->value("Lucene Port");
  155. debugbr("acquired Axyl config: host=$this->host, port=$this->port");
  156. }
  157. }
  158. // Try to open socket if we have a host..
  159. $this->connected = false;
  160. if ($this->enabled && $this->host != "") {
  161. $this->sockfp = fsockopen($this->host, $this->port);
  162. if(!$this->sockfp) {
  163. $this->log_error("failed to connect to '$this->host:$this->port'");
  164. }
  165. else {
  166. if ($timeoutsecs != "") $this->timeoutsecs = $timeoutsecs;
  167. $this->set_timeout($this->timeoutsecs);
  168. $this->connected = true;
  169. debugbr("lucene_connection: connected to '$this->host:$this->port'");
  170. }
  171. }
  172. // Return result..
  173. return $this->connected;
  174. } // connect
  175. // .....................................................................
  176. /**
  177. * Disconnect from the Lucene server. Normally this is used only by
  178. * internal Luceneserver methods.
  179. * @access private
  180. */
  181. function disconnect() {
  182. if ($this->connected) {
  183. fclose($this->sockfp);
  184. $this->sockfp = false;
  185. }
  186. } // disconnect
  187. // .....................................................................
  188. /**
  189. * Set the socket timeout. Deals with the special case of setting
  190. * the socket to non-blocking mode (zero timeout)..
  191. * @param integer $timeoutsecs Set the timeout in seconds
  192. */
  193. function set_timeout($timeoutsecs) {
  194. if ($this->connected && $timeoutsecs != "") {
  195. $this->timeoutsecs = $timeoutsecs;
  196. if ($this->timeoutsecs != SOCK_NO_WAIT) {
  197. socket_set_timeout( $this->sockfp, $this->timeoutsecs);
  198. }
  199. socket_set_blocking( $this->sockfp, (($this->timeoutsecs == SOCK_NO_WAIT) ? false : true) );
  200. }
  201. } // set_timeout
  202. // .....................................................................
  203. /**
  204. * Sends a message to the Lucene server, and receives the response. We
  205. * operate on the understanding that every time we send something to
  206. * Lucene we expect a response. Since this method already calls the
  207. * recieve() method, there is no need to call it from your application.
  208. * The content to be sent is expected to be already in the class
  209. * string variable $message. The response is put into $response which
  210. * is an array of LF-delimited lines sent back.
  211. * @param integer $timeoutsecs Override for timeout in seconds
  212. * @return boolean True if the message was sent ok
  213. */
  214. function send($timeoutsecs="") {
  215. $send_ok = true;
  216. $this->response = array();
  217. if (!$this->connected) {
  218. $this->connect();
  219. }
  220. if ($this->connected) {
  221. // Check for timeout over-ride..
  222. if ($timeoutsecs != "") $this->timeoutsecs = $timeoutsecs;
  223. $this->set_timeout($this->timeoutsecs);
  224. // Send message..
  225. if ($this->message != "") {
  226. $this->timer->restart();
  227. $bytesput = fputs($this->sockfp, $this->message);
  228. $this->timer->stop();
  229. if (debugging()) {
  230. $buf = trim(substr(rawurldecode($this->message),0, 5000));
  231. debugbr("<pre>" . xmldump($buf) . "</pre>", DBG_DUMP);
  232. debugbr("lucene_connection: send transaction took " . $this->timer->formatted_millisecs() . "mS");
  233. }
  234. if ($bytesput != -1) {
  235. debugbr("lucene_connection: send ok ($bytesput bytes)");
  236. for ($i=0; $i< SOCK_RETRIES; $i++) {
  237. $send_ok = $this->receive();
  238. if ($send_ok) break;
  239. debugbr("lucene_connection: receive retry #" . ($i + 1));
  240. }
  241. }
  242. else {
  243. $this->log_error("write to server failed");
  244. $send_ok = false;
  245. }
  246. }
  247. else {
  248. $this->log_error("trying to send null content");
  249. $send_ok = false;
  250. }
  251. }
  252. else {
  253. $this->log_error("send with no open socket");
  254. $send_ok = false;
  255. }
  256. // Return status..
  257. return $send_ok;
  258. } // send
  259. // .....................................................................
  260. /**
  261. * Receive a message from the Lucene server. We can specify a timeout
  262. * period in seconds. If set to SOCK_NO_WAIT, it will return immediately with or
  263. * without a message. This is a low-level routine which deals with receiving the
  264. * message over TCP sockets.
  265. * @return boolean True if the message was received loud and clear
  266. * @access private
  267. */
  268. function receive() {
  269. $received_ok = true;
  270. if ($this->connected) {
  271. $this->timer->restart();
  272. $this->responsebuf = "";
  273. while (!feof($this->sockfp)) {
  274. $buf = fread($this->sockfp, 10000);
  275. if ($buf !== false) {
  276. $this->responsebuf .= $buf;
  277. }
  278. else {
  279. $this->log_error("no response from server");
  280. $received_ok = false;
  281. break;
  282. }
  283. }
  284. $this->timer->stop();
  285. if (debugging()) {
  286. debugbr("<pre>" . xmldump($this->responsebuf) . "</pre>", DBG_DUMP);
  287. debugbr("lucene_connection: response from server took " . $this->timer->formatted_millisecs() . "mS");
  288. }
  289. }
  290. else {
  291. $this->log_error("receive with no open socket");
  292. $received_ok = false;
  293. }
  294. // Return status..
  295. return $received_ok;
  296. } // receive
  297. // .....................................................................
  298. /** Log a message to the syslog and print info to debugger.
  299. * @access private
  300. */
  301. function log_error($err) {
  302. $prefix = (defined("APP_NAME") ? APP_NAME . ": " : "");
  303. $err = "Lucene error: " . get_class($this) . ": $this->host:$this->port: $err";
  304. debugbr($err);
  305. error_log($prefix . $err, 0);
  306. } // log_error
  307.  
  308. } // lucene_connection class
  309. // ----------------------------------------------------------------------
  310.  
  311. /** The lucene fieldset class. This holds the Lucene fields for a lucene
  312. * message. These fields comprise the list of tags which make up
  313. * a query message or an index message.
  314. * @access private
  315. * @package search
  316. */
  317. class lucene_fieldset {
  318. /** Fields stored as an array of XML <Field> tags */
  319.  
  320. var $xmltags = array();
  321. // .....................................................................
  322. /** Constructor */
  323.  
  324. function lucene_fieldset() { }
  325. // .....................................................................
  326. /**
  327. * Return a copy of the named field object from fieldset by name.
  328. * NOTES: This function will return a new field if it does not already
  329. * exist. In this case the field will not be stored until you use the
  330. * put() method to do so. Always returns a field object.
  331. * @param string $fieldname The name of the field to get
  332. * @return object An xmltag object for the field
  333. */
  334. function get_field($fieldname) {
  335. if (isset($this->xmltags[$fieldname])) {
  336. $field = $this->xmltags[$fieldname];
  337. }
  338. else {
  339. $field = new xmltag("Field");
  340. $field->setattribute("name", $fieldname);
  341. }
  342. return $field;
  343. } // get_field
  344. // .....................................................................
  345. /**
  346. * Puts the named field into fieldset, indexed by fieldname.
  347. * @param string $fieldname Unique name of the field in the set
  348. * @param object $field The field object to store
  349. */
  350. function put_field($fieldname, $field) {
  351. $this->xmltags[$fieldname] = $field;
  352. } // put_field
  353. // .....................................................................
  354. /** Define a field in the fieldset. Set the definition for a field
  355. * in this fieldset. If the field does not exist it is created and
  356. * its definition set. If it exists the definition is updated.
  357. * @param string $fieldname Name of the field
  358. * @param string $type Type of this field eg. "Date"
  359. * @param boolean $stored Whether field value should be stored by Lucene
  360. * @param boolean $indexed Whether field value should be indexed by Lucene
  361. */
  362. function define_field($fieldname, $type, $stored=STORED, $indexed=INDEXED) {
  363. $field = $this->get_field($fieldname);
  364. $field->setattribute("type", $type);
  365. $field->setattribute("stored", ($stored ? "true" : "false"));
  366. $field->setattribute("indexed", ($indexed ? "true" : "false"));
  367. $this->put_field($fieldname, $field);
  368. } // define_field
  369. // .....................................................................
  370. /** Add a field to the fieldset.
  371. * @param string $fieldname Name of the field
  372. * @param string $fieldvalue Value to associate with this field
  373. */
  374. function add_field($fieldname, $fieldvalue="") {
  375. $field = $this->get_field($fieldname);
  376. $field->value = $fieldvalue;
  377. $this->put_field($fieldname, $field);
  378. } // add_field
  379. // .....................................................................
  380. /** Clear all fields from the fieldset */
  381.  
  382. function clear() {
  383. $this->xmltags = array();
  384. } // clear
  385. // .....................................................................
  386. function render() {
  387. $s = "";
  388. foreach ($this->xmltags as $field) {
  389. $s .= $field->render();
  390. }
  391. return $s;
  392. } // render
  393.  
  394. } // lucene_fieldset class
  395. // ----------------------------------------------------------------------
  396.  
  397. /**
  398. * The lucene msg class. This is a raw class which holds the basic
  399. * message fields and data and knows how to build them into a full
  400. * message for sending to the lucene server.
  401. * @package search
  402. */
  403. class lucene_msg extends lucene_connection {
  404. // Public
  405. /** Type/name of this message */
  406.  
  407. var $type = "";
  408.  
  409. // Private
  410. /** Array containing XML tags
  411. @access private */
  412. var $xmltags = array();
  413. /** Object containing lucene fields
  414. @access private */
  415. var $fieldset;
  416. /** True if message has been built
  417. @access private */
  418. var $built = false;
  419. /** Error message if any error occurred
  420. @access private */
  421. var $error_msg = "";
  422. // .....................................................................
  423. /** Constructor
  424. * Notes: The application is either specified in the formal paramters or it
  425. * can be determined for an Axyl application by using the APP_PREFIX which
  426. * is unique to the application. This is the recommended option. Other
  427. * developers have, however, also used the configvalue 'Lucene Application'
  428. * for some reason, so this is still supported here. If none of these
  429. * methods results in a valid identifier, 'default' is used.
  430. * @param string $type Type of message this is, eg; QUERY, INDEX..
  431. * @param string $application The application name. Sets default Lucene config.
  432. * @param string $host Hostname or IP of Lucene server
  433. * @param string $port Port of Lucene server
  434. */
  435. function lucene_msg($type="", $application="?", $host="", $port="") {
  436. $this->lucene_connection($host, $port);
  437. $this->type = $type;
  438. $this->fieldset = new lucene_fieldset();
  439. // We must have an application..
  440. if ($application == "?") {
  441. if (class_exists("configuration")) {
  442. $config = new configuration("sys_control");
  443. $application = $config->value("Lucene Application");
  444. }
  445. // Axyl configuration value may not be defined and
  446. // the APP_PREFIX will be used in this case..
  447. if ($application == "" || $application == "?") {
  448. if ( defined("APP_PREFIX")) {
  449. $application = APP_PREFIX;
  450. }
  451. else {
  452. // The default case for standalone apps..
  453. $application = "default";
  454. }
  455. }
  456. }
  457. // Set the application..
  458. $this->set_application($application);
  459. } // lucene_msg
  460. // .....................................................................
  461. /**
  462. * Add a new XML tag object to this Lucene message
  463. * @param object $tag Tha xmltag object to add to our lucene msg
  464. */
  465. function add_xmltag($tag) {
  466. $this->xmltags[] = $tag;
  467. $this->built = false;
  468. } // add_xmltag
  469. // .....................................................................
  470. /**
  471. * Specify the application. The application is the name of a configuration
  472. * set which has been specified either by a control message, or by using
  473. * configuration files on the server. A given configuration set identified
  474. * by an application name can have specific fields already defined, such
  475. * as Sort: or Domain: etc.
  476. * Notes: The 'Application' header can only appear once in the message.
  477. * @param string $application The application name to set.
  478. */
  479. function set_application($application) {
  480. $this->add_xmltag( new xmltag("Application", $application) );
  481. } // set_application
  482. // .....................................................................
  483. /**
  484. * Specify a domain. A domain is an identifier which groups indexed
  485. * objects internally to Lucene. This allows searches on multiple
  486. * archives of documents in a single Lucene installation.
  487. * Notes: There may be zero or more domain headers in the message. If it
  488. * does not appear, then any domain header defined for the application
  489. * will be applied on its own. Otherwise any definitions added by this
  490. * method are OR'd with any specified in the application config.
  491. * NB: If no domains are specified anywhere, any searching will be done
  492. * across all domains (which would probably yield very confusing return
  493. * data!).
  494. * @param string $domain The domain to set.
  495. */
  496. function set_domain($domain) {
  497. $this->add_xmltag( new xmltag("Domain", $domain) );
  498. } // set_domain
  499. // .....................................................................
  500. /** Add a field to the fieldset.
  501. * @param string $fieldname Name of the field
  502. * @param string $fieldvalue Value to associate with this field
  503. */
  504. function add_field($fieldname, $fieldvalue="") {
  505. $this->fieldset->add_field($fieldname, $fieldvalue);
  506. $this->built = false;
  507. } // add_field
  508. // .....................................................................
  509. /** Clear all data/fields, leaving type definition alone. */
  510.  
  511. function clear() {
  512. $this->fieldset->clear();
  513. $this->message = "";
  514. $this->built = false;
  515. } // clear
  516. // .....................................................................
  517. /**
  518. * Builds the message according to the message type. This method
  519. * may be over-ridden in children inheriting this class
  520. * @access private
  521. */
  522. function build() {
  523. if (!$this->built) {
  524. if ($this->type != "") {
  525. $xml = new xmltag($this->type);
  526. // XML TAGS
  527. foreach ($this->xmltags as $tag) {
  528. $xml->childtag($tag);
  529. }
  530. // FIELDS
  531. if (count($this->fieldset->xmltags) > 0) {
  532. $fields = new xmltag("Fields");
  533. foreach ($this->fieldset->xmltags as $field) {
  534. $fields->childtag($field);
  535. }
  536. $xml->childtag($fields);
  537. }
  538. $this->message = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" . $xml->render();
  539. $this->built = true;
  540. }
  541. }
  542. return $this->built;
  543. } // build
  544. // .....................................................................
  545. /**
  546. * Sends the current message to Lucene, and checks for protocol
  547. * errors in the received response.
  548. * @param integer $timeoutsecs Override for timeout in seconds
  549. */
  550. function send($timeoutsecs="") {
  551. if ($this->build()) {
  552. // Low-level socket send-receive transaction..
  553. lucene_connection::send($timeoutsecs);
  554. // Once a msg is sent, socket can be closed..
  555. $this->disconnect();
  556. }
  557. } // send
  558.  
  559. } // lucene_msg class
  560. // ----------------------------------------------------------------------
  561.  
  562. /**
  563. * The lucene message class. This class extends its parent class
  564. * lucene_msg and adds some higher level methods for adding groups of
  565. * fields to the message.
  566. * @package search
  567. */
  568. class lucene_message extends lucene_msg {
  569. /** Response object which will parse XML content
  570. @access private */
  571. var $response;
  572. // .....................................................................
  573. /** Constructor
  574. * This is a more complex class which builds on the basic lucene_msg
  575. * class to provide some higher level methods for adding fields in
  576. * specific ways to support CONTROL, QUERY and INDEX message types.
  577. * @param string $type Type of message this is, eg; QUERY, INDEX..
  578. * @param string $application The application name. Sets default Lucene config.
  579. * @param string $host Hostname or IP of Lucene server
  580. * @param string $port Port of Lucene server
  581. */
  582. function lucene_message($type="", $application="?", $host="", $port="") {
  583. $this->lucene_msg($type, $application, $host, $port);
  584. } // lucene_message
  585. // .....................................................................
  586. /**
  587. * Strip field type specifiers out of field strings. A field string with
  588. * a type specifier in it is of the form: 'Foo:Date', where the field
  589. * name is 'Foo' and the field type is 'Date'. Possible field types are
  590. * 'Id', 'Text' (the default), and 'Date'.
  591. * Note that sort field specification is a special case, where the syntax
  592. * can be 'Foo:Date:Desc' or 'Foo:Desc' indicating the sort on the given
  593. * field should be done in descending order.
  594. * At present you would only use this facility with a 'Date' field, and
  595. * everything else would then default to 'Text'. [The 'Id' type being a
  596. * special one]
  597. * We return the field stripped of any type, and if a type was present
  598. * we issue the define_field() directive to define it. A field so-defined
  599. * will always be both stored by Lucene and indexed.
  600. * @param string $field Field in 'Foo:Date' format, or just 'Foo' for default type
  601. * @return string The fieldname stripped of any type specifier
  602. * @access private
  603. */
  604. function strip_field_type($field) {
  605. $fieldname = $field;
  606. $retfieldname = $field;
  607. if (strstr($field, ":")) {
  608. // Extract field specifier parts..
  609. $bits = explode(":", $field);
  610. $fieldname = trim( array_shift($bits) );
  611. $retfieldname = $fieldname;
  612. $f1 = trim(array_shift($bits));
  613. $f2 = trim(array_shift($bits));
  614. // Check for a sort field with DESC specifier..
  615. if ($f1 == "Desc" || $f2 == "Desc") {
  616. $retfieldname .= ":Desc";
  617. }
  618. // Check for valid field type specifier..
  619. if ($f1 == "Date" || $f1 == "Text" || $f1 == "Id") {
  620. // Define field by name..
  621. $this->define_field($fieldname, $f1);
  622. }
  623. }
  624. // Return fieldname plus any sort spec..
  625. return $retfieldname;
  626. } // strip_field_type
  627. // .....................................................................
  628. /**
  629. * Define a field. We supply the name of the field, it's type (Text, Date
  630. * or Id), and whether it should be stored by Lucene for later retreival
  631. * in queries. For example you would not store the raw document/content as
  632. * this is usually stored elsewhere.
  633. * We also cater for fields which might not need to be indexed. These would
  634. * be fields of data you just want to return with the document, if found in
  635. * a query, but not search on. An example might be a field containing the
  636. * path to the physical document on disk. For these fields you would then
  637. * specify NOT_INDEXED for the $indexed parameter. These fields MUST be
  638. * stored, so we make the rule: if the field is NOT_INDEXED then it must
  639. * be STORED (this will be forced).
  640. * In the normal course of events, fields will be defined to be both stored
  641. * and indexed. The exception is the special "Text" field associated with
  642. * an item "Body", which is indexed, but never stored.
  643. * This method adds the field settings directly via the add_field() method.
  644. * @see add_field()
  645. * @param string $fieldname Name of the field to index
  646. * @param string $type Type of field data: Text, Date or Id.
  647. * @param boolean $stored If true then Lucene will store the content itself
  648. * @param boolean $indexed If true then Lucene will index the field content
  649. */
  650. function define_field($fieldname, $type, $stored=STORED, $indexed=INDEXED) {
  651. // Force non-indexed fields to be stored..
  652. if ($indexed == NOT_INDEXED) $stored = STORED;
  653. $this->fieldset->define_field($fieldname, $type, $stored, $indexed);
  654. } // define_field
  655. // .....................................................................
  656. /**
  657. * Specify the fields you want returned from Lucene.
  658. * Fields should be in a comma-separated list of field names. Each field
  659. * name can have the field type included in the form 'Foo:Date', where
  660. * 'Date' is the type in this instance. In fact, since 'Text' is the
  661. * default filed type, 'Date' is probably the only one you need to use
  662. * as the current implementation stands.
  663. * This method adds the field setting directly via the add_field() method.
  664. * @see add_field
  665. * @param mixed $fields Comma-delimited fieldname list, or array of fields
  666. */
  667. function set_returnfields($fields) {
  668. if (!is_array($fields)) {
  669. $flds = explode(",", $fields);
  670. }
  671. else {
  672. $flds = $fields;
  673. }
  674. $returnfields = array();
  675. foreach ($flds as $field) {
  676. $returnfields[] = $this->strip_field_type($field);
  677. }
  678. $returnlist = implode(" ", $returnfields);
  679. $this->add_xmltag( new xmltag("Return", $returnlist) );
  680. } // set_returnfields
  681. // .....................................................................
  682. /**
  683. * Specify query limit field. This sets the maximum number of results
  684. * that Lucene should return.
  685. * @param integer $limit Maximum number of results (hits) to return
  686. */
  687. function set_limit($limit) {
  688. $this->add_xmltag( new xmltag("Limit", $limit) );
  689. } // set_limit
  690. // .....................................................................
  691. /**
  692. * Specify query offset field 'First'. This sets the offset for the
  693. * returned results. For example, if this was set to 3, and Lucene
  694. * found 20 hits, then results would be sent back from the 3rd hit
  695. * onwards.
  696. * @param integer $first Offset in result set to start from
  697. */
  698. function set_first($first) {
  699. $this->add_xmltag( new xmltag("First", $first) );
  700. } // set_first
  701. // .....................................................................
  702. /**
  703. * Specify the fields you want query results to be ordered by.
  704. * Fields should be in a comma-separated list of field names. Each field
  705. * name can have the field type included in the form 'Foo:Date', where
  706. * 'Date' is the type in this instance. In fact, since 'Text' is the
  707. * default filed type, 'Date' is probably the only one you need to use
  708. * as the current implementation stands.
  709. * Note that sort field specification is a special case, where the syntax
  710. * can be 'Foo:Date:Desc' or 'Foo:Desc' indicating the sort on the given
  711. * field should be done in descending order.
  712. * @param mixed $fields Comma-delimited fieldname list, or array of fields
  713. */
  714. function set_sortorder($fields) {
  715. if (!is_array($fields)) {
  716. $flds = explode(",", $fields);
  717. }
  718. else {
  719. $flds = $fields;
  720. }
  721. $sortfields = array();
  722. foreach ($flds as $field) {
  723. $sortfields[] = $this->strip_field_type($field);
  724. }
  725. // Create the field..
  726. $sortlist = implode(" ", $sortfields);
  727. $this->add_xmltag( new xmltag("Sort", $sortlist) );
  728. } // set_sortorder
  729. // .....................................................................
  730. /**
  731. * Specify a range on a field for querying. We specify the name of a field
  732. * which is used to select articles within the given limits, and
  733. * the limits themeselves. Either limit may be passed as nullstring
  734. * which indicates no limit on that side. Any dates must be passed as
  735. * standard Unix timestamps (seconds since 1970).
  736. * Notes: This method can be called multiple times to define additional
  737. * ranges for different field names.
  738. * This method adds the field setting directly via the add_field() method.
  739. * @see add_field
  740. * @param string $range_from Value of lowerbound range
  741. * @param string $range_to Value of upperbound range
  742. * @param string $range_fieldname Name of field to use in range query.
  743. */
  744. function set_range($range_from="", $range_to="", $range_fieldname="") {
  745. if ($range_fieldname != "") {
  746. $range = new xmltag("Range");
  747. $range->setattribute("field", $this->strip_field_type($range_fieldname));
  748. if ($range_from != "" && $range_from != false) {
  749. $range->childtag( new xmltag("From", $range_from) );
  750. }
  751. if ($range_to != "" && $range_to != false) {
  752. $range->childtag( new xmltag("To", $range_to) );
  753. }
  754. $this->add_xmltag( $range );
  755. }
  756. } // set_range
  757. // .....................................................................
  758. /**
  759. * Supply a stopword list to lucene.
  760. * This method adds the field setting directly via the add_field() method.
  761. * @see add_field
  762. * @param mixed $stopwords Space-delimited list, or array of stopwords
  763. */
  764. function set_stopwords($stopwords) {
  765. if (is_array($stopwords)) {
  766. $mystops = implode(" ", $stopwords);
  767. }
  768. else {
  769. $mystops = $stopwords;
  770. }
  771. $this->add_xmltag( new xmltag("Stop-List", $mystops) );
  772. } // set_stopwords
  773.  
  774. } // lucene_message class
  775. // ----------------------------------------------------------------------
  776.  
  777. /**
  778. * Encapsulation of the result of a generic search query. This is for
  779. * internal use only.
  780. * @package search
  781. * @access private
  782. */
  783. class queryresult {
  784. var $rank = "";
  785. var $fields = array();
  786.  
  787. function queryresult($rank="") {
  788. $this->rank = $rank;
  789. }
  790. function addfield($fieldname, $fieldvalue) {
  791. $this->fields[$fieldname] = $fieldvalue;
  792. }
  793. } // queryresult class
  794. // ----------------------------------------------------------------------
  795.  
  796. /**
  797. * Class comprising the functionality of a Lucene response parser. This
  798. * is for internal use only.
  799. * @package search
  800. * @access private
  801. */
  802. class response_parser extends xmlparser {
  803. /** Current/last tag opened */
  804.  
  805. var $tag = "";
  806. /** Attributes array for current/last tag */
  807.  
  808. var $attr = array();
  809. /** Serial transaction ID */
  810.  
  811. var $serial = "";
  812. /** Status message */
  813.  
  814. var $status_message = "";
  815. /** True if response was valid, ie. no errors */
  816.  
  817. var $valid = true;
  818. // .....................................................................
  819. /** Construct a new parser. */
  820.  
  821. function response_parser() {
  822. $this->xmlparser();
  823. } // response_parser
  824. // .....................................................................
  825. /** Method invoked when a tag is opened */
  826.  
  827. function tag_open($parser, $tag, $attributes) {
  828. $this->tag = $tag;
  829. if (is_array($attributes) && count($attributes) > 0) {
  830. foreach ($attributes as $key => $value ) {
  831. $this->attr[$key] = $value;
  832. }
  833. }
  834. switch ($tag) {
  835. case "Error":
  836. $this->valid = false;
  837. break;
  838. } // switch
  839. } // tag_open
  840. // .....................................................................
  841. /** Method invoked when character data is available */
  842.  
  843. function cdata($parser, $cdata) {
  844. switch ($this->tag) {
  845. case "Error":
  846. $this->error_message = $cdata;
  847. debugbr("lucene error: $this->error_message");
  848. break;
  849. case "Status":
  850. $this->status_message = $cdata;
  851. debugbr("lucene status: $this->status_message");
  852. break;
  853. case "Serial":
  854. $this->serial = $cdata;
  855. break;
  856. } // switch
  857. } // cdata
  858. // .....................................................................
  859. /** Method invoked when a tag is closed */
  860.  
  861. function tag_close($parser, $tag) {
  862. $this->tag = "";
  863. $this->attr = array();
  864. } // tag_close
  865. // .....................................................................
  866. function parse($xml) {
  867. xmlparser::parse($xml);
  868. if (!$this->valid_xml) {
  869. $this->valid = false;
  870. }
  871. if ($this->error_message != "") {
  872. log_sys($this->error_message);
  873. }
  874. } // parse
  875.  
  876. } // response_parser class
  877. // ----------------------------------------------------------------------
  878.  
  879. /**
  880. * Class comprising the functionality of an XML parser for queries. This
  881. * is for internal use only.
  882. * @package search
  883. * @access private
  884. */
  885. class queryresponse_parser extends response_parser {
  886. /** Results returned count */
  887.  
  888. var $count = 0;
  889. var $results;
  890. var $results_stream = false;
  891. // .....................................................................
  892. /** Construct a new parser. */
  893.  
  894. function queryresponse_parser() {
  895. $this->response_parser();
  896. } // queryresponse_parser
  897. // .....................................................................
  898. /** Method invoked when a tag is opened */
  899.  
  900. function tag_open($parser, $tag, $attributes) {
  901. response_parser::tag_open($parser, $tag, $attributes);
  902. switch ($tag) {
  903. case "Results":
  904. $this->results_stream = true;
  905. break;
  906. case "Result":
  907. $this->addresult(
  908. $this->attr["counter"],
  909. $this->attr["rank"]
  910. );
  911. $this->attr = array();
  912. break;
  913. } // switch
  914. } // tag_open
  915. // .....................................................................
  916. /** Method invoked when character data is available */
  917.  
  918. function cdata($parser, $cdata) {
  919. response_parser::cdata($parser, $cdata);
  920. switch ($this->tag) {
  921. case "Count":
  922. $this->count = $cdata;
  923. break;
  924. case "Field":
  925. if ($this->results_stream) {
  926. if (count($this->attr) > 0) {
  927. $result = array_pop($this->results);
  928. $fieldname = $this->attr["name"];
  929. $fieldval = $cdata;
  930. $result->addfield($fieldname, $fieldval);
  931. array_push($this->results, $result);
  932. }
  933. $this->attr = array();
  934. }
  935. break;
  936. } // switch
  937. } // cdata
  938. // .....................................................................
  939. /** Method invoked when a tag is closed */
  940.  
  941. function tag_close($parser, $tag) {
  942. response_parser::tag_close($parser, $tag);
  943. switch ($tag) {
  944. case "Results":
  945. $this->results_stream = false;
  946. break;
  947. } // switch
  948. } // tag_close
  949. // .....................................................................
  950. /** Add a result field to the response */
  951.  
  952. function addresult($id, $rank) {
  953. $this->results[$id] = new queryresult($rank);
  954. } // addresult
  955.  
  956. } // queryresponse_parser class
  957. // ----------------------------------------------------------------------
  958.  
  959. /**
  960. * The lucene query message class. This class inherits all the functionality
  961. * of the lucene_connection, lucene_msg and lucene_message classes. It adds
  962. * query-specific methods for searching.
  963. * @package search
  964. */
  965. class lucene_querymsg extends lucene_message {
  966. /** Set to true if sort limit was exceeded in query */
  967.  
  968. var $sort_limit_exceeded = false;
  969. /** Set to true if Lucene blew its memory trying to sort */
  970.  
  971. var $sort_memory_exceeded = false;
  972. // .....................................................................
  973. /** Constructor
  974. * Make a new Lucene query message. You can specify the application to
  975. * use here, and also an optional query string to send.
  976. * @param string $application Optional application specifier.
  977. * @param string $host Hostname or IP of Lucene server
  978. * @param string $port Port of Lucene server
  979. */
  980. function lucene_querymsg($application="?", $host="", $port="") {
  981. $this->lucene_message("LuceneQueryRequest", $application, $host, $port);
  982. } // lucene_querymsg
  983. // .....................................................................
  984. /**
  985. * Set the query for this message. There can be only one query defined.
  986. * This method can be called repeatedly, and each time it is called the
  987. * new value will replace the old one.
  988. * @param string $query The query to submit to Lucene.
  989. */
  990. function set_query($query) {
  991. $queryxml = new xmltag("Query", $query);
  992. $queryxml->setattribute("default-field", DEFAULT_FIELD);
  993. $this->add_xmltag($queryxml);
  994. } // set_query
  995. // .....................................................................
  996. /**
  997. * Send the message to Lucene, and then post-process the response for
  998. * query hits. The hitcount is extracted, followed by the hits, which
  999. * may comprise multiple fields. A hit is thus defined as an array of
  1000. * fields, and each hit is put into a single container array called
  1001. * 'hit', which is a property of the parent class 'search'.
  1002. * @param integer $timeoutsecs Override for timeout in seconds
  1003. */
  1004. function send($timeoutsecs="") {
  1005. // Initialise flags..
  1006. $this->sort_limit_exceeded = false;
  1007. $this->sort_memory_exceeded = false;
  1008.  
  1009. // Msg-level send-receive transaction..
  1010. lucene_message::send($timeoutsecs);
  1011.  
  1012. // Process the response to our request..
  1013. $this->response = new queryresponse_parser();
  1014. $this->response->parse($this->responsebuf);
  1015.  
  1016. // Unpack the response if no errors..
  1017. if ($this->response->valid) {
  1018. // Here we will unpack the returned search query hits
  1019. // and store them locally for use by child classes.
  1020. $this->hitcount = (int)($this->response->count);
  1021. if (isset($this->response->results)) {
  1022. foreach ($this->response->results as $result) {
  1023. $hit = array();
  1024. $hit["RANK"] = $result->rank;
  1025. foreach ($result->fields as $fieldname => $fieldvalue) {
  1026. $hit[$fieldname] = $fieldvalue;
  1027. }
  1028. $this->hit[] = $hit;
  1029. }
  1030. }
  1031. }
  1032. else {
  1033. // Check for sort limit/memory error conditions..
  1034. if (stristr($this->response->error_message, "system sort limit")) {
  1035. $this->sort_limit_exceeded = true;
  1036. }
  1037. if (stristr($this->response->error_message, "out of memory")) {
  1038. $this->sort_memory_exceeded = true;
  1039. }
  1040. }
  1041. } // send
  1042.  
  1043. } // lucene_querymsg class
  1044. // ----------------------------------------------------------------------
  1045.  
  1046. /**
  1047. * The lucene index message class. This class inherits all the functionality
  1048. * of the lucene_connection, lucene_msg and lucene_message classes. It adds
  1049. * indexing-specific methods.
  1050. * @package search
  1051. */
  1052. class lucene_indexmsg extends lucene_message {
  1053. // Public
  1054. /** Indication that the indexing was successful */
  1055.  
  1056. var $indexed = false;
  1057.  
  1058. // Private
  1059. /** A unique handle to identify the index
  1060. response from Lucene
  1061. @access private */
  1062. var $serialno = "";
  1063. // .....................................................................
  1064. /** Constructor
  1065. * Make a new Lucene index message.
  1066. * @param string $application Optional application specifier
  1067. * @param string $host Hostname or IP of Lucene server
  1068. * @param string $port Port of Lucene server
  1069. */
  1070. function lucene_indexmsg($application="?", $host="", $port="") {
  1071. $this->lucene_message("LuceneIndexRequest", $application, $host, $port);
  1072. $this->serialno = md5(uniqid(""));
  1073. $this->add_xmltag( new xmltag("Serial", $this->serialno) );
  1074. $this->define_field(DEFAULT_FIELD, DEFAULT_FIELDTYPE, NOT_STORED);
  1075. } // lucene_indexmsg
  1076. // .....................................................................
  1077. /**
  1078. * Supply field content for indexing. This causes Lucene to take the given
  1079. * fieldname and index the given value against it. NB: we silently ignore
  1080. * the request for nullstring, since these cause Lucene indexing to throw
  1081. * an exception, and indexing will fail.
  1082. * The field name can have the field type included in the form 'Foo:Date',
  1083. * where 'Date' is the type in this instance. In fact, since 'Text' is the
  1084. * default filed type, 'Date' is probably the only one you need to use
  1085. * as the current implementation stands.
  1086. * @param string $fieldname Name of the field to index.
  1087. * @param string $fieldvalue Content of the field to index
  1088. */
  1089. function index_field($fieldname, $fieldvalue) {
  1090. if ($fieldvalue !== "") {
  1091. $fieldname = $this->strip_field_type($fieldname);
  1092. $this->add_field($fieldname, $fieldvalue);
  1093. }
  1094. } // index_field
  1095. // .....................................................................
  1096. /**
  1097. * Index the given content against the given ID. This automatically
  1098. * defines the default field called "Text", and the data added as a field
  1099. * called "Text" as well. Attaches the "Body" tag to this field via a
  1100. * call to add_data() method. Thus, the content is submitted as a raw
  1101. * binary stream, rather than url-encoded text.
  1102. * @param string $id The ID to associate with the given indexed data.
  1103. * @param string $content The binary/text content to be indexed.
  1104. */
  1105. function index_content($id, $content) {
  1106. if ($content !== "") {
  1107. $this->add_xmltag( new xmltag("Id", $id) );
  1108. $content = preg_replace("/[\n\r\t]/", " ", $content);
  1109. $content = preg_replace("/[ ]{2,}/", " ", $content);
  1110. $this->add_field(DEFAULT_FIELD, $content);
  1111. }
  1112. } // index_content
  1113. // .....................................................................
  1114. /**
  1115. * Send the message to Lucene, and then post-process the response for
  1116. * indication of a successful index operation. We expect to receive
  1117. * a response back from Lucene which has our serialno in it. This method
  1118. * returns True if the indexing was successful, else False.
  1119. * @param integer $timeoutsecs Override for timeout in seconds
  1120. * @return boolean True if indexing was successful.
  1121. */
  1122. function send($timeoutsecs="") {
  1123. // Msg-level send-receive transaction..
  1124. lucene_message::send($timeoutsecs);
  1125.  
  1126. // Process the response to our request..
  1127. $this->response = new response_parser();
  1128. $this->response->parse($this->responsebuf);
  1129.  
  1130. // Unpack the response if no errors..
  1131. if ($this->response->valid) {
  1132. $this->indexed = ($this->response->serial == $this->serialno);
  1133. }
  1134. // Return status of indexing operation..
  1135. return $this->indexed;
  1136. } // send
  1137.  
  1138. } // lucene_indexmsg class
  1139. // ----------------------------------------------------------------------
  1140.  
  1141. /**
  1142. * The lucene unindex message class. This class allows you to remove an
  1143. * item from the Lucene index. You must know the unique ID that identifies
  1144. * the document.
  1145. * @package search
  1146. */
  1147. class lucene_unindexmsg extends lucene_message {
  1148. // .....................................................................
  1149. /** Constructor
  1150. * Make a new Lucene unindex message. This message is provided to allow
  1151. * you to delete an item from the Lucene index. It has a single method
  1152. * 'unindex' which takes the ID of the item to delete.
  1153. * @param string $application Optional application specifier
  1154. * @param string $host Hostname or IP of Lucene server
  1155. * @param string $port Port of Lucene server
  1156. */
  1157. function lucene_unindexmsg($application="?", $host="", $port="") {
  1158. $this->lucene_message("LuceneUnIndexRequest", $application, $host, $port);
  1159. } // lucene_unindexmsg
  1160. // .....................................................................
  1161. /**
  1162. * Unindex the given document, as identified by the unique ID. If no errors
  1163. * arise, then the item will be removed from the Lucene index.
  1164. * @param string $id The ID to allow Lucene to identify the item to unindex
  1165. */
  1166. function unindex($id) {
  1167. $this->add_xmltag( new xmltag("Id", $id) );
  1168. } // unindex
  1169.  
  1170. } // lucene_unindexmsg class
  1171. // ----------------------------------------------------------------------
  1172.  
  1173. /**
  1174. * The lucene purge message class. This class allows you to remove all
  1175. * items from the Lucene index. Take care!
  1176. * @package search
  1177. */
  1178. class lucene_purgemsg extends lucene_message {
  1179. // .....................................................................
  1180. /** Constructor
  1181. * Make a new Lucene purge message. This message is provided to allow
  1182. * you to delete all items from the Lucene index.
  1183. * @param string $application Optional application specifier
  1184. * @param string $host Hostname or IP of Lucene server
  1185. * @param string $port Port of Lucene server
  1186. */
  1187. function lucene_purgemsg($application="?", $host="", $port="") {
  1188. $this->lucene_message("LuceneUnIndexRequest", $application, $host, $port);
  1189. $this->add_xmltag( new xmltag("Purge") );
  1190. } // lucene_purgemsg
  1191.  
  1192. } // lucene_purgemsg class
  1193. // ----------------------------------------------------------------------
  1194.  
  1195. /**
  1196. * The lucene utility message class. Used for special Lucene operations.
  1197. * @package search
  1198. */
  1199. class lucene_utilitymsg extends lucene_message {
  1200. /** Constructor
  1201. * @param string $utilitycmd Command for this utility message.
  1202. * @param string $application Optional application specifier
  1203. * @param string $host Hostname or IP of Lucene server
  1204. * @param string $port Port of Lucene server
  1205. */
  1206. function lucene_utilitymsg($utilitycmd="", $application="?", $host="", $port="") {
  1207. $this->lucene_message("LuceneUtilityRequest", $application, $host, $port);
  1208. if ($utilitycmd != "") {
  1209. $this->add_xmltag( new xmltag("Utility", $utilitycmd) );
  1210. }
  1211. } // lucene_utilitymsg
  1212. // .....................................................................
  1213. /**
  1214. * Send the message to Lucene, and then post-process the response for
  1215. * indication of a successful utility operation. We expect to receive
  1216. * a response back from Lucene which has nothing much it, unless there
  1217. * has been an error.
  1218. * returns True if the operation was successful, else False.
  1219. * @param integer $timeoutsecs Override for timeout in seconds
  1220. * @return boolean True if operation was successful.
  1221. */
  1222. function send($timeoutsecs="") {
  1223. // Msg-level send-receive transaction..
  1224. lucene_message::send($timeoutsecs);
  1225.  
  1226. // Process the response to our request..
  1227. $this->response = new response_parser();
  1228. $this->response->parse($this->responsebuf);
  1229.  
  1230. // Return status of indexing operation..
  1231. return $this->response->valid;
  1232. } // send
  1233.  
  1234. } // lucene_utilitymsg class
  1235. // ----------------------------------------------------------------------
  1236.  
  1237. /**
  1238. * The lucene search class
  1239. * This class inherits the functionality of the generic 'search' class. It
  1240. * extends it to implement a LUCENE search. Use the methods in this class
  1241. * as the mainstay in implementing queries of content from Lucene. Most
  1242. * methods, such as match(), matchfield(), matchrange() etc. store the
  1243. * requirement in the class for subsequent building using the set_*()
  1244. * methods of the lucene classes to set the relevant fields. This is only
  1245. * done when you call execute(), and the query is built from all the
  1246. * composite terms you have added via match() et al.
  1247. * @package search
  1248. */
  1249. class lucene_search extends lucene_querymsg {
  1250. // .....................................................................
  1251. /**
  1252. * Constructor
  1253. * Create a new lucene search
  1254. * @param string $application Application name/domain name for searching in
  1255. * @param string $host Hostname or IP of Lucene server
  1256. * @param string $port Port of Lucene server
  1257. */
  1258. function lucene_search($application="?", $host="", $port="") {
  1259. $this->search();
  1260. $this->lucene_querymsg($application, $host, $port);
  1261. $this->initialise();
  1262. } // lucene_search
  1263. // .....................................................................
  1264. /**
  1265. * Add a new search term to match. Search terms can be a single word or
  1266. * compound patterns, Each time one of these is added, it has an operator
  1267. * associated with it - whether this term is a "may have" (OR), or a
  1268. * "must have" (AND) term.
  1269. * NB: This method overrides the parent method in order to ensure that all
  1270. * boolean logic terms are in upper case as Lucene requires.
  1271. * @param string $term Search term text to match.
  1272. * @param integer $op Joining operator: 'AND', 'OR', 'NOT, 'AND NOT'.
  1273. * @param string $id An optional ID to associate with this search term.
  1274. * @param numeric $boost Boost factor. Can be a fraction eg. 0.2, or integer 1,2,3..
  1275. */
  1276. function match($term, $op="OR", $id="", $boost="") {
  1277. $LCops = array("/ and /","/ or /","/ not /");
  1278. $UCops = array(" AND "," OR "," NOT ");
  1279. $term = preg_replace($LCops, $UCops, $term);
  1280. if ($boost != "") $term .= "^$boost";
  1281. search::match($term, strtoupper($op), $id);
  1282. } // match
  1283. // .....................................................................
  1284. /**
  1285. * Add search term to match a field value.
  1286. * This is used to add a search term which defines the value that a given
  1287. * field may or may not contain for the search to succeed.
  1288. * For adding terms which are 'free' (as a user might type into a search
  1289. * box for example) then you can use the match() method which this class
  1290. * inherits from the search class.
  1291. * @param string $fieldname Name of field to reference in the index
  1292. * @param mixed $fieldvalue Value or array of values, for field to match
  1293. * @param string $op Operator to join this term to others in the query
  1294. * @param string $id Optional identity tag for this term
  1295. * @param numeric $boost Boost factor. Can be a fraction eg. 0.2, or integer 1,2,3..
  1296. */
  1297. function matchfield($fieldname, $fieldvalue, $op="OR", $id="", $boost="") {
  1298. debug_trace($this);
  1299. if (!isset($fieldvalue)) return;
  1300. if (!is_array ($fieldvalue)) {
  1301. $fieldvalue = array($fieldvalue);
  1302. }
  1303. $term = "";
  1304. foreach ($fieldvalue as $value) {
  1305. $value = trim($value);
  1306. if ($value != "") {
  1307. $term .= " OR " . $this->fieldterm($fieldname, $value);
  1308. }
  1309. }
  1310. if ($term != "") {
  1311. $term = substr($term, 4); // Get rid of initial OR
  1312. // Call parent function to register the search term..
  1313. $this->match($term, strtoupper($op), $id, $boost);
  1314. }
  1315. debug_trace();
  1316. } // matchfield
  1317. // .....................................................................
  1318. /**
  1319. * Helper function to build field search term
  1320. * @param string $fieldname Name of field to reference in the index
  1321. * @param string $fieldvalue Value of field to match
  1322. * @access private
  1323. */
  1324. function fieldterm($fieldname, $fieldvalue) {
  1325. if ($fieldname != DEFAULT_FIELD) {
  1326. $term = "$fieldname:$fieldvalue";
  1327. }
  1328. else {
  1329. $term = $fieldvalue;
  1330. }
  1331. return $term;
  1332. } // fieldterm
  1333. // .....................................................................
  1334. /**
  1335. * Add search term to match a field value range.
  1336. * This is used to add a search term which defines the range of values that
  1337. * a given field may or may not contain for the search to succeed.
  1338. * NB: This method is always a must match (implied AND) search term. In
  1339. * other words the search is always restricted/refined by it.
  1340. * @param string $fromvalue Lower range value of field to match
  1341. * @param string $tovalue Upper range value of field to match
  1342. * @param string $fieldname Name of field, defaulted to 'Text'
  1343. */
  1344. function matchrange($fromvalue, $tovalue, $fieldname) {
  1345. debug_trace($this);
  1346. $this->set_range($fromvalue, $tovalue, $fieldname);
  1347. debug_trace();
  1348. } // matchrange
  1349. // .....................................................................
  1350. /**
  1351. * Add search term: must match a field value.
  1352. * This is used to add a search term which defines the value that a given
  1353. * field must contain for the search to succeed.
  1354. * @param string $fieldname Name of field
  1355. * @param string $fieldvalue Value of field to match
  1356. * @param string $id Optional identity tag for this term
  1357. * @param numeric $boost Boost factor. Can be a fraction eg. 0.2, or integer 1,2,3..
  1358. */
  1359. function must_matchfield($fieldname, $fieldvalue, $id="", $boost="") {
  1360. $this->matchfield($fieldname, $fieldvalue, "AND", $id, $boost);
  1361. } // must_matchfield
  1362. // .....................................................................
  1363. /**
  1364. * Add search term: may match a field value.
  1365. * This is used to add a search term which defines the value that a given
  1366. * field may contain for the search to succeed.
  1367. * @param string $fieldname Name of field
  1368. * @param string $fieldvalue Value of field to match
  1369. * @param string $id Optional identity tag for this term
  1370. * @param numeric $boost Boost factor. Can be a fraction eg. 0.2, or integer 1,2,3..
  1371. */
  1372. function may_matchfield($fieldname, $fieldvalue, $id="", $boost="") {
  1373. $this->matchfield($fieldname, $fieldvalue, "OR", $id, $boost);
  1374. } // may_matchfield
  1375. // .....................................................................
  1376. /**
  1377. * Add search term: must not match a field value.
  1378. * This is used to add a search term which defines the value that a given
  1379. * field must not contain for the search to succeed.
  1380. * @param string $fieldname Name of field
  1381. * @param string $fieldvalue Value of field to match
  1382. * @param string $id Optional identity tag for this term
  1383. * @param numeric $boost Boost factor. Can be a fraction eg. 0.2, or integer 1,2,3..
  1384. */
  1385. function does_not_matchfield($fieldname, $fieldvalue, $id="", $boost="") {
  1386. $this->matchfield($fieldname, $fieldvalue, "NOT", $id, $boost);
  1387. } // does_not_matchfield
  1388. // .....................................................................
  1389. /**
  1390. * Execute the search
  1391. * Here we execute a lucene search, overriding the method in the parent
  1392. * class. This involves building the query string, sending it to the
  1393. * Lucene server, and receiving the search results back.
  1394. * @param integer $timeoutsecs Override for timeout in seconds
  1395. */
  1396. function execute($timeoutsecs="") {
  1397. debug_trace($this);
  1398.  
  1399. // The queryvalid() method is in the parent class 'search', and
  1400. // calls the build() method in the same class. The build() method is
  1401. // a raw routine to join together the search terms with ANDs and
  1402. // ORs. You may have to override it for Lucene. If so, just create
  1403. // a new build() method in this class.
  1404.  
  1405. if ($this->queryvalid()) {
  1406.  
  1407. // Define the query string..
  1408. $this->set_query($this->query);
  1409.  
  1410. // Set limit, offset..
  1411. if ($this->max_results > 0) {
  1412. $this->set_limit($this->max_results);
  1413. if ($this->skip_results > 0) {
  1414. $this->set_first($this->skip_results);
  1415. }
  1416. }
  1417.  
  1418. // Set any daterange..
  1419. if ($this->has_daterange()) {
  1420. $this->set_range($this->date_start, $this->date_end, $this->date_fieldname);
  1421. }
  1422.  
  1423. // Send to Lucene..
  1424. $this->send($timeoutsecs);
  1425.  
  1426. // Flag that we did it..
  1427. $this->executed = true;
  1428. debugbr("lucene search: exec ok: returning " . $this->hitcount() . " hits");
  1429. }
  1430. else {
  1431. debugbr("lucene search: invalid query: '$this->query'");
  1432. }
  1433. debug_trace();
  1434. } // execute
  1435.  
  1436. } // lucene_search class
  1437. // ----------------------------------------------------------------------
  1438.  
  1439. /**
  1440. * The lucene file indexer class.
  1441. * This class indexes files on disc, either one by one or as a whole
  1442. * file hierarchy tree.
  1443. * @package search
  1444. */
  1445. class lucene_fileindexer {
  1446. // Public
  1447. /** Application we are indexing for */
  1448.  
  1449. var $application = "";
  1450. /** Host to connect to */
  1451.  
  1452. var $host = "";
  1453. /** Port to connect to */
  1454.  
  1455. var $port = "";
  1456.  
  1457. // Private
  1458. /** The index ID
  1459. @access private */
  1460. var $ixid;
  1461. /** ID generation source
  1462. @access private */
  1463. var $idsource = ID_FROM_INC;
  1464. /** Scan for meta tags as fields in file content. Recommended.
  1465. @access private */
  1466. var $metascan = true;
  1467. /** Meta fields definitions array. Contains definitions
  1468. for the fields we will process if found as meta tags.
  1469. @access private */
  1470. var $meta_fields = array();
  1471. /** Index fields definitions array. Contains definitions
  1472. for the fields we are expecting to index.
  1473. @access private */
  1474. var $field_definitions = array();
  1475. /** Fields for indexing. This is an array of fieldname/value
  1476. pairs which should be added during the indexing. These
  1477. fields do not have to appear in $field_definitions.
  1478. @access private */
  1479. var $indexfields = array();
  1480. /** ID generation offset
  1481. @access private */
  1482. var $idoffset = 0;
  1483. /** ID generation prefix
  1484. @access private */
  1485. var $idprefix = "";
  1486. /** The index object which does the work
  1487. @access private */
  1488. var $lucene_indexer;
  1489. /** Timeout for indexing commands in seconds (can usually leave
  1490. as nullstring)
  1491. @access private */
  1492. var $timeoutsecs = "";
  1493. /** Path to a lockfile we should give way to. If this value
  1494. is not nullstring, then no indexing will be done while the
  1495. file exists. If lockfile_wait is > 0, then we only wait
  1496. this many seconds.
  1497. @access private */
  1498. var $lockfile = "";
  1499. /** Number of seconds to wait on a lockfile. If zero, wait forever.
  1500. @access private */
  1501. var $lockfile_wait_secs = 0;
  1502. /** Indexing execution timer
  1503. @access private */
  1504. var $timer;
  1505. // .....................................................................
  1506. /**
  1507. * Constructor
  1508. * Create a new lucene indexer
  1509. * @param string $application Application name
  1510. * @param string $host Hostname or IP of Lucene server
  1511. * @param string $port Port of Lucene server
  1512. */
  1513. function lucene_fileindexer($application="?", $host="", $port="") {
  1514. // Store for reference..
  1515. $this->application = $application;
  1516. $this->host = $host;
  1517. $this->port = $port;
  1518. $this->timer = new microtimer();
  1519. } // lucene_fileindexer
  1520. // .....................................................................
  1521. /**
  1522. * Define a field. We supply the name of the field, it's type (Text, Date
  1523. * or Id), and whether it should be stored by Lucene for later retreival in
  1524. * queries. For example you would not store the raw document/content as this
  1525. * is usually stored elsewhere.
  1526. * IMPORTANT NOTE: Fields defined here will automatically be included as
  1527. * meta fields.
  1528. * @see meta_fields()
  1529. * @param string $fieldname Name of the field to index
  1530. * @param string $type Type of field data: Text, Date or Id.
  1531. * @param boolean $stored If true then Lucene will store the content itself
  1532. * @param boolean $indexed If true then Lucene will index the field content
  1533. */
  1534. function define_field($fieldname, $type, $stored=STORED, $indexed=INDEXED) {
  1535. $this->field_definitions[$fieldname]
  1536. = $type . "|" . (($stored) ? "true" : "false") . "|" . (($indexed) ? "true" : "false");
  1537. // Register for meta tags..
  1538. $this->meta_field($fieldname, $type);
  1539. } // define_field
  1540. // .....................................................................
  1541. /**
  1542. * Define a lockfile which we must avoid during indexing. If defined
  1543. * then no indexing will take place while the lockfile exists. The
  1544. * second parameter allows you to specify a limit to the patience of
  1545. * this process, in seconds. Zero means wait forever.
  1546. * @param string $lockfile Path to the lockfile. Nullstring = not defined
  1547. * @param integer $wait_secs Time to wait for lockfile. Zero means forever.
  1548. */
  1549. function avoid_lockfile($lockfile, $wait_secs=0) {
  1550. $this->lockfile = $lockfile;
  1551. $this->lockfile_wait_secs = $wait_secs;
  1552. } // avoid_lockfile
  1553. // .....................................................................
  1554. /**
  1555. * Define a field as a meta tag. This ensures that the field will be
  1556. * picked up from the file meta tags, if present. If it is not listed
  1557. * here then it will be ignored.
  1558. * IMPORTANT NOTE: We define the strict rule that ONLY fields which have
  1559. * been defined here can be added to the indexing via the meta tag scanning.
  1560. * Ie. you must define fields here explicitly, or via the define_field()
  1561. * method, or they will be ignored even if they turn up as a meta tag.
  1562. * This is so we can restrict the indexing, and be sure of field types.
  1563. * @see define_field()
  1564. * @param string $fieldname Name of the field to process as meta tag
  1565. * @param string $type Type of field data: Text, Date or Id.
  1566. */
  1567. function meta_field($fieldname, $type) {
  1568. $this->meta_fields[$fieldname] = $type;
  1569. } // meta_field
  1570. // .....................................................................
  1571. /**
  1572. * Supply field content for indexing. This causes Lucene to take the given
  1573. * fieldname and index the given value against it.
  1574. * The field name can have the field type included in the form 'Foo:Date',
  1575. * where 'Date' is the type in this instance. In fact, since 'Text' is the
  1576. * default filed type, 'Date' is probably the only one you need to use
  1577. * as the current implementation stands.
  1578. * @param string $fieldname Name of the field to index.
  1579. * @param string $fieldvalue Content of the field to index
  1580. */
  1581. function index_field($fieldname, $fieldvalue) {
  1582. $this->indexfields[$fieldname] = $fieldvalue;
  1583. } // index_field
  1584. // .....................................................................
  1585. /**
  1586. * Set the source for ID generation. Since we are indexing a bunch of
  1587. * files, the ID's have to be generated on demand inside the loop. So
  1588. * we provide for various ways here, and you can extend this class to
  1589. * provide more if required.
  1590. * Main ways:
  1591. * ID_FROM_INC Increment a counter by 1 each time (with offset)
  1592. * ID_FROM_NAME Take the filename, strip the extension, add prefix
  1593. * ID_FROM_FILENAME Take the full filename, add prefix
  1594. * ID_FROM_PATH Take the full file path
  1595. * NB: These are all defined as integer constants.
  1596. * @param integer $idsource Source of ID generation
  1597. * @param mixed $pfxofs String prefix, or integer offset
  1598. */
  1599. function id_generate($idsource=ID_FROM_INC, $pfxofs="") {
  1600. $this->idsource = $idsource;
  1601. if ($pfxofs != "") {
  1602. if (is_string($pfxofs)) {
  1603. $this->idprefix = $pfxofs;
  1604. }
  1605. else {
  1606. $this->idoffset = (int)$pfxofs;
  1607. }
  1608. }
  1609. } // id_generate
  1610. // .....................................................................
  1611. /**
  1612. * Flag that we should do a tag scan on the content of the files to try
  1613. * and extract fields to index. Note that any tags thus found will only
  1614. * be used if the field name has been defined with the method define_field();
  1615. * This causes both the <title> tag and <meta> tags to be considered.
  1616. * @see lucene_fileindexer::define_field()
  1617. */
  1618. function scantags() {
  1619. $this->metascan = true;
  1620. } // scantags
  1621. // .....................................................................
  1622. /**
  1623. * Flag that we should NOT do a tag scan on the content of the files.
  1624. */
  1625. function noscantags() {
  1626. $this->metascan = false;
  1627. } // noscantags
  1628. // .....................................................................
  1629. /**
  1630. * Index a file located at the given path, using given ID.
  1631. * You can also use the parameter $fields to supply an array of
  1632. * fieldname/value pairs to index with this file, for one-off indexing of
  1633. * files. If the fieldname is a date field, make sure to define the
  1634. * name as 'Foo:Date', to cause the field definition to be correct.
  1635. * @param string $path Path to the head of the file tree to index
  1636. * @param string $id ID to associate with the indexed file content
  1637. * @param mixed $fields Array of field/values to index with file
  1638. */
  1639. function index_file($path, $id, $fields=false) {
  1640. $success = false;
  1641. $f = new inputfile($path);
  1642. if ($f->opened) {
  1643. $f->readall();
  1644. $f->closefile();
  1645.  
  1646. // Wait for a lockfile, if we really have to..
  1647. if ($this->lockfile != "" && file_exists($this->lockfile)) {
  1648. $waitforit = true;
  1649. debugbr("waiting for lockfile..", DBG_DEBUG);
  1650. if ($this->lockfile_wait_secs > 0) {
  1651. $locktimer = new microtimer();
  1652. $locktimer->start();
  1653. }
  1654. do {
  1655. clearstatcache();
  1656. if (!file_exists($this->lockfile)) {
  1657. $waitforit = false;
  1658. debugbr("lockfile has been removed..", DBG_DEBUG);
  1659. }
  1660. elseif ($this->lockfile_wait_secs > 0 && $locktimer->secs() >= $this->lockfile_wait_secs) {
  1661. $waitforit = false;
  1662. debugbr("lockfile wait (" . $this->lockfile_wait_secs ."secs) timed out..", DBG_DEBUG);
  1663. }
  1664. else {
  1665. sleep(1);
  1666. }
  1667. } while ($waitforit === true);
  1668. }
  1669.  
  1670. // Create the index message..
  1671. $ix = new lucene_indexmsg($this->application, $this->host, $this->port);
  1672.  
  1673. // Define the fields for the index message..
  1674. foreach ($this->field_definitions as $fieldname => $attributes) {
  1675. $bits = explode("|", $attributes);
  1676. $type = $bits[0];
  1677. $stored = (strcasecmp($bits[1], "true") == 0);
  1678. $indexed = (strcasecmp($bits[2], "true") == 0);
  1679. $ix->define_field($fieldname, $type, $stored, $indexed);
  1680. }
  1681.  
  1682. // Scan file content for meta tags for index fields..
  1683. $content = preg_replace("/[\xe2][\x80]./", "", $f->content);
  1684. $content = preg_replace("/[\xc2][\xb7]./", "", $content);
  1685. $content = preg_replace("/[\xc2]&/", " ", $content);
  1686. $content = preg_replace("/[\xc3]&/", " ", $content);
  1687.  
  1688. if ($this->metascan) {
  1689. $tagpat = "/<meta name=\"(.*?)\" content=\"(.*?)\">/i";
  1690. $matches = array();
  1691. if (preg_match_all($tagpat, $content, $matches)) {
  1692. for ($i=0; $i < count($matches[0]); $i++) {
  1693. $fieldname = $matches[1][$i];
  1694. $fieldvalue = $matches[2][$i];
  1695. if (isset($this->meta_fields[$fieldname])) {
  1696. // Get type..
  1697. $type = $this->meta_fields[$fieldname];
  1698. if (!strcasecmp($type, "date")) {
  1699. // Newsquest date field format requires stripping off a prefix
  1700. // 'DT' - a temporary hack which should be completely transparent
  1701. // to everyone else using this. NB: originally NewsQuest only
  1702. // stored date in 'DTdd/mm/yyyy' format. This parsing is also
  1703. // compatible with the new 'DTdd/mm/yyyy hh:mm[:ss]' format.
  1704. if (substr($fieldvalue, 0, 2) == "DT") {
  1705. $fieldvalue = substr($fieldvalue, 2);
  1706. }
  1707. // Need to convert to Unix timestamp..
  1708. $ts = displaydate_to_timestamp($fieldvalue);
  1709. $fieldvalue = $ts;
  1710. }
  1711. debugbr("meta tag index field: $fieldname=$fieldvalue");
  1712. $ix->index_field($fieldname, $fieldvalue);
  1713. }
  1714. else {
  1715. debugbr("rejected unlisted tag field: $fieldname");
  1716. }
  1717. }
  1718. }
  1719. // Check for title tag in HTML page if required field..
  1720. if (preg_match("/<(title)>(.*?)<\/title>/i", $content, $matches)) {
  1721. $fieldname = $matches[1];
  1722. $fieldvalue = $matches[2];
  1723. if (isset($this->meta_fields[$fieldname])) {
  1724. $type = $this->meta_fields[$fieldname];
  1725. debugbr("title tag index field: $fieldname=$fieldvalue");
  1726. $ix->index_field($fieldname, $fieldvalue);
  1727. }
  1728. }
  1729. } // metascan
  1730.  
  1731. // Deal with passed-in field settings. These are meant to cater
  1732. // for indexing of individual files using this method. We just
  1733. // add them to any existing field/values already set up..
  1734. if ($fields) {
  1735. reset($fields);
  1736. while (list($fieldname, $fieldvalue) = each($fields)) {
  1737. $this->index_field($fieldname, $fieldvalue);
  1738. }
  1739. }
  1740.  
  1741. // Process field/value pairs which have been added either by the
  1742. // index_field() method, or passed in via the $fields parameter..
  1743. if (count($this->indexfields) > 0) {
  1744. reset($this->indexfields);
  1745. while (list($fieldname, $fieldvalue) = each($this->indexfields)) {
  1746. $bits = explode(":", $fieldname);
  1747. $type = ((isset($bits[1])) ? $bits[1] : "Text");
  1748. $fieldname = $bits[0];
  1749. debugbr("index field: $fieldname=$fieldvalue");
  1750. $ix->define_field($fieldname, $type);
  1751. $ix->index_field($fieldname, $fieldvalue);
  1752. }
  1753. }
  1754.  
  1755. // Index the file content. We get rid of any HTML tags..
  1756. debugbr("indexing file: $path, ID=$id");
  1757. $ix->index_content($id, strip_tags($content));
  1758.  
  1759. // Send the index message to lucene. We specify a large
  1760. // timeout since we really want this to succeed and Lucene
  1761. // may be in an optimization fugue..
  1762. $success = $ix->send(120);
  1763. if(!$success) {
  1764. debugbr("failed: $ix->error_msg");
  1765. }
  1766. }
  1767. else {
  1768. debugbr("open failed on '$path'");
  1769. }
  1770. return $success;
  1771. } // index_file
  1772. // .....................................................................
  1773. /**
  1774. * Index a tree of files starting at the path given. We index these in one
  1775. * of four modes, which determines how we generate the ID for each item:
  1776. * 'ID_FROM_INC' mode uses an incremental counter starting at 1. If $prefix
  1777. * holds a number, the counter will start at this number instead of one.
  1778. * Each item has an ID incremented by one from the last one.
  1779. * 'ID_FROM_NAME' mode uses the filename, stripped of any path and extension
  1780. * as the ID. If prefix is not a nullstring, then it is prefixed to every
  1781. * filename ID.
  1782. * 'ID_FROM_FILENAME' mode uses the filename, including any extension
  1783. * as the ID. If prefix is not a nullstring, then it is prefixed to every
  1784. * filename ID.
  1785. * 'ID_FROM_PATH' mode uses the full path to the item being indexed as the
  1786. * ID. If prefix is not a nullstring, then it is prefixed to every
  1787. * filename ID.
  1788. * The file will simply be indexed as a single Text field, with the
  1789. * appropriate ID, and no other index fields unless $metascan is set to TRUE.
  1790. * If this is the case, the system will scan the file for HTML meta tags of
  1791. * form: '<meta name="foo" content="bar">'. In this example a field of name
  1792. *'foo' would be given value 'bar'.
  1793. * @param string $path Path to the head of the file tree to index
  1794. * @param $patt Pattern to match, eg. '*.html'
  1795. * @param $restart If equal to "restart" then treat $path as file of paths
  1796. * @param $lockfile If path is set, we idle whilst this file exists
  1797. * @param string $lockfile Path to the lockfile. Nullstring = not defined
  1798. * @param integer $wait_secs Time to wait for lockfile. Zero means forever.
  1799. */
  1800. function index_tree($path, $patt="", $restart="", $lockfile="", $wait_secs=0) {
  1801. // Set up any lockfile definition..
  1802. $this->avoid_lockfile($lockfile, $lockfile_wait_secs);
  1803.  
  1804. if ($restart == "restart") {
  1805. // Restart from existing paths file..
  1806. $tmpfname = $path;
  1807. debugbr("restarting with existing item list $path", DBG_DEBUG);
  1808. }
  1809. else {
  1810. // Use find to generate item list to a temporary file..
  1811. debugbr("generating item list", DBG_DEBUG);
  1812. $tmpfname = tempnam("/tmp", "LU");
  1813. $cmd = "find $path";
  1814. if ($patt != "") $cmd .= " -name \"$patt\"";
  1815. $cmd .= " >$tmpfname";
  1816. exec($cmd);
  1817. }
  1818. $treelist = new inputfile($tmpfname);
  1819. if ($treelist->opened) {
  1820. // Find the number of items..
  1821. debugbr("counting items", DBG_DEBUG);
  1822. $todo = (int) exec("cat $tmpfname|wc -l");
  1823. if ($todo > 0) {
  1824. $done = 0; $succeeded = 0; $failed = 0; $last = 0;
  1825. debugbr("$todo items to index", DBG_DEBUG);
  1826. $this->timer->start();
  1827. $idix = 0;
  1828. if ($this->idsource == ID_FROM_INC) {
  1829. $idix += $this->idoffset;
  1830. }
  1831.  
  1832. while ($path = $treelist->readln()) {
  1833. // Generate an ID to use..
  1834. switch ($this->idsource) {
  1835. case ID_FROM_INC:
  1836. // Use incremented index..
  1837. $id = $idix + 1;
  1838. $idix += 1;
  1839. break;
  1840.  
  1841. case ID_FROM_NAME:
  1842. // Use filename, minus extenaion..
  1843. $fname = basename($path);
  1844. if (strstr($fname, ".")) {
  1845. $bits = explode(".", $fname);
  1846. $dummy = array_pop($bits);
  1847. $fname = implode(".", $bits);
  1848. }
  1849. $id = $this->idprefix . $fname;
  1850. break;
  1851.  
  1852. case ID_FROM_FILENAME:
  1853. // Use full filename..
  1854. $id = $this->idprefix . basename($path);
  1855. break;
  1856.  
  1857. case ID_FROM_PATH:
  1858. // Use full file path..
  1859. $id = $this->idprefix . $path;
  1860. break;
  1861. } // switch
  1862.  
  1863. // Index the file with new ID..
  1864. if ($this->index_file($path, $id)) {
  1865. debugbr("$id indexed", DBG_DEBUG);
  1866. $succeeded += 1;
  1867. }
  1868. else {
  1869. debugbr("$path index failed", DBG_DEBUG);
  1870. //break;
  1871. $failed += 1;
  1872. }
  1873.  
  1874. // Progress check..
  1875. $done += 1;
  1876.  
  1877. // If the verbose output option is enabled, we compile
  1878. // stats and display these via the debugger..
  1879. if (debugging()) {
  1880. $pct = ($done / $todo) * 100;
  1881. $pct_int = (int)(floor($pct));
  1882. $pct_mod = $pct % 5;
  1883. if ($pct_mod == 0 && $pct_int > $last) {
  1884. $secperdoc = $this->timer->secs() / $done;
  1885. $timedone = $this->timer->formatted_time();
  1886. $timeleft = nicetime(($todo - $done) * $secperdoc);
  1887. $ms = $this->timer->millisecs();
  1888. $msper = number_format( ($ms / $done), 0);
  1889. debugbr("Mark: $pct_int% $timedone ($done) Rate:$msper" . "ms/item Left:$timeleft", DBG_DEBUG);
  1890. $last = $pct_int;
  1891. }
  1892. }
  1893. } // while
  1894.  
  1895. // Close tree list file..
  1896. $treelist->closefile();
  1897.  
  1898. // Wrap it up..
  1899. $this->timer->stop();
  1900.  
  1901. // Final stats if verbose mode..
  1902. if (debugging()) {
  1903. $secs = $this->timer->secs();
  1904. $msper = number_format( (1000 * $secs / $todo), 2);
  1905. $sper1000 = number_format( ($secs / $todo) * 1000, 2);
  1906. debugbr("time taken per item: " . $msper . "msec", DBG_DEBUG);
  1907. debugbr("time per 1000 items: " . nicetime($sper1000), DBG_DEBUG);
  1908. debugbr("total time taken: " . $this->timer->formatted_time(), DBG_DEBUG);
  1909. debugbr("successfully indexed: $succeeded", DBG_DEBUG);
  1910. debugbr("indexing failures: $failed", DBG_DEBUG);
  1911. }
  1912. }
  1913. else {
  1914. debugbr("nothing to index", DBG_DEBUG);
  1915. }
  1916. }
  1917. else {
  1918. debugbr("failed to open $tmpfname", DBG_DEBUG);
  1919. }
  1920. } // index_tree
  1921.  
  1922. } // lucene_fileindexer class
  1923. // ----------------------------------------------------------------------
  1924.  
  1925. /**
  1926. * Function to optimize the Lucene index. This would commonly
  1927. * be used after a batch of items have been indexed.
  1928. * @param string $application Application name/domain name for searching in
  1929. * @param string $host Hostname or IP of Lucene server
  1930. * @param string $port Port of Lucene server
  1931. * @return boolean True if the operation was successful.
  1932. */
  1933. function lucene_optimize($application="?", $host="", $port="") {
  1934. $optimizer = new lucene_utilitymsg("OPTIMIZE", $application, $host, $port);
  1935. $optimizer->send(SOCK_FOREVER);
  1936. return $optimizer->response->valid;
  1937. } // lucene_optimize
  1938. // ----------------------------------------------------------------------
  1939.  
  1940. /**
  1941. * Function to make a backup of the Lucene index. This would commonly
  1942. * be used after a batch of items have been successfully optimized (which
  1943. * indicates a sound index). The backup will be made to the directory
  1944. * specified in the application .properties file as the property
  1945. * 'Lucene-Backup-Directory=' or, if not there then in the Lucene properties
  1946. * file 'Server.properties' as the same property. If neither of these are
  1947. * defined, the server will attempt to use a sub-directory called
  1948. * {Lucene-Index-Directory}_backup, where {Lucene-Index-Directory} is the
  1949. * index path as already defined in the 'Server.properties' file.
  1950. * @param string $application Application name/domain name for searching in
  1951. * @param string $host Hostname or IP of Lucene server
  1952. * @param string $port Port of Lucene server
  1953. * @return boolean True if the operation was successful.
  1954. */
  1955. function lucene_backup($application="?", $host="", $port="") {
  1956. $backup = new lucene_utilitymsg("BACKUP", $application, $host, $port);
  1957. $backup->send(SOCK_FOREVER);
  1958. return $backup->response->valid;
  1959. } // lucene_backup
  1960. // ----------------------------------------------------------------------
  1961.  
  1962. /**
  1963. * Function to purge the Lucene index of all indexes to documents. Yes,
  1964. * I'll repeat that - it DELETES ALL DOCUMENTS FROM THE INDEX, permanently,
  1965. * finito, shazam, ba-boom, as in "Omigod did I *really* mean to do that!?".
  1966. * I guess I don't have to warn you to be careful with this, do I?
  1967. * @param string $application Application name/domain name for searching in
  1968. * @param string $host Hostname or IP of Lucene server
  1969. * @param string $port Port of Lucene server
  1970. * @return boolean True if the purging operation was successful.
  1971. */
  1972. function lucene_purge($application="?", $host="", $port="") {
  1973. $purgative = new lucene_purgemsg($application, $host, $port);
  1974. $purgative->send(SOCK_FOREVER);
  1975. return $purgative->response->valid;
  1976. } // lucene_purge
  1977. // ----------------------------------------------------------------------
  1978.  
  1979. ?>

Documentation generated by phpDocumentor 1.3.0RC3