Go Back   EQEmulator Home > EQEmulator Forums > Development > Development::Development

Development::Development Forum for development topics and for those interested in EQEMu development. (Not a support forum)

Reply
 
Thread Tools Display Modes
  #1  
Old 10-28-2008, 03:19 AM
AndMetal
Developer
 
Join Date: Mar 2007
Location: Ohio
Posts: 648
Default

First, I would like to clarify, I have been talking about loading the spells into shared memory, not loading info on the fly from the database (that's just resource suicide).

Quote:
Originally Posted by KLS View Post
-You have slightly increased load times for spells.
I know PHP isn't the best "language" for speed, but according to preliminary testing, it actually IS faster to calculate the max spell ID & load the spells from the database than the spells_us.txt file WITHOUT caching (SQL_NO_CACHE):

PHP Code:
<?php

class Database
{
    private 
$connectoid false//Used to track this specific database connection so we don't mess with the other connections, also can be used to determine the state of the connection
    
private static $user "username";    //Username to access the Database
    
private static $pass "password";    //Password to access the Database
    
private static $host "localhost";        //IP address or hostname
    
protected $db "peq";    //Name of the Database we're accessing, can be changed if needed
    
    
protected $benchmarks = Array();
    
    public function 
__construct() {
        if (
$this->connectoid mysql_connect(Database::$hostDatabase::$userDatabase::$pass)) {
            
mysql_select_db($this->db$this->connectoid);
        }
    }
    
    public function 
__toString() {
        return 
"Database " . ($this->IsConnected() ? "" "not ") . "connected";
    }
    
    public function 
IsConnected() {
        return (
$this->connectoid true false);
    }
    
    
//test function, we don't want to be broadcasting our login info to everyone irl
    
public function ReturnLoginInfo() {
        return 
"Host: " Database::$host "<br>\nUser: " Database::$user "<br>\nPass: " Database::$pass "<br>\nDB: " $this->db;
    }
    
    public function 
Query($q) {
        
$this->benchmarks[Query][result][start] = microtime(true);
        
$res mysql_query($q$this->connectoid);    //results
        
$this->benchmarks[Query][result][end] = microtime(true);
        
$this->benchmarks[Query][row][start] = microtime(true);
        if (
$row mysql_fetch_array($resMYSQL_ASSOC)) {    //single row of data
            
$this->benchmarks[Query][row][end] = microtime(true);
            
$i 0;    //iterator
            
$ret = array();
            do {
                
$ret[++$i] = $row;
            } while (
$row mysql_fetch_array($resMYSQL_ASSOC));
            if (
is_array($ret))    //Only return data in an array, otherwise it's an error
                
return $ret;
        } else {
            return 
$this->GetErrorNum($this->connectoid);
        }
    }
    
    public function 
GetBenchmarks() {
        
$this->TotalBenchmarks();
        return 
$this->benchmarks;
    }
    
    private function 
TotalBenchmarks() {
        
$this->benchmarks[Query][result][total] = $this->benchmarks[Query][result][end] - $this->benchmarks[Query][result][start];
        
$this->benchmarks[Query][row][total] = $this->benchmarks[Query][row][end] - $this->benchmarks[Query][row][start];
    }
    
    protected function 
GetErrorNum($c) {
        return 
mysql_errno($c);
    }
    
    public function 
__destruct() {
        if (
$this->IsConnected())
            
mysql_close($this->connectoid);
    }
}

?>
PHP Code:
<html>

<?php

ini_set
("memory_limit","256M");
ini_set("max_execution_time","600");

//Automagically load Classes where needed from ../includes/ClassName.class.php
function __autoload($class_name) {
    require_once 
'../includes/' $class_name '.class.php';
}

function 
DatabaseGetMax() {

    
$db_time_start microtime(true);
    
$db = new Database;
    
$db_time_query microtime(true);
    
$db_spells $db->Query("SELECT SQL_NO_CACHE MAX(id) FROM spells_new");
    
$db_time_end microtime(true);
    
    
$db_time_total_overhead $db_time_query $db_time_start;
    
$db_time_total_query $db_time_end $db_time_query;
    
$db_time_total_mysql $db->GetBenchmarks();
    
$db_time_total $db_time_end $db_time_start;
    echo 
"Result: " $db_spells[1]["MAX(id)"] . "<br>\n";
    echo 
"Class Time: " $db_time_total_overhead " seconds<br>\n";
    echo 
"Query Time: " $db_time_total_query " seconds (MySQL Time: " $db_time_total_mysql[Query][result][total] . " seconds)<br>\n";
    echo 
"Total Time: " $db_time_total " seconds<br>\n";

}

function 
DatabaseLoad() {

    
$db_time_start microtime(true);
    
$db = new Database;
    
$db_time_query microtime(true);
    
$db_spells $db->Query("SELECT SQL_NO_CACHE * FROM spells_new");
    
$db_time_end microtime(true);
    
    
$db_time_total_overhead $db_time_query $db_time_start;
    
$db_time_total_query $db_time_end $db_time_query;
    
$db_time_total_mysql $db->GetBenchmarks();
    
$db_time_total $db_time_end $db_time_start;
    echo 
"Spells Loaded: " count($db_spells) . "<br>\n";
    echo 
"Class Time: " $db_time_total_overhead " seconds<br>\n";
    echo 
"Query Time: " $db_time_total_query " seconds (MySQL Time: " $db_time_total_mysql[Query][result][total] . " seconds)<br>\n";
    echo 
"Total Time: " $db_time_total " seconds<br>\n";

}

function 
FileGetMax() {

    
$file_dir "";    // needs / at the end if anything other than the current directory
    
$file_name "spells_us.txt";
    
    
$file_time_start microtime(true);
    if (
file_exists($file_dir $file_name) && is_readable($file_dir $file_name)) {

        
$file_time_load microtime(true);
        
$Lines file($file_dir $file_name);
        
        
$file_time_parse microtime(true);
        foreach(
$Lines as $key => $value) {
            
$Data[$key] = explode("^"$value);
        }

        
$file_time_end microtime(true);

    } else {
        echo 
"Unable to read: " $file_dir $file_name "<br>\n";
    }
    
    if (!
$file_time_end)
        
$file_time_end microtime(true);
    
    
$file_time_total_open $file_time_load $file_time_start;
    
$file_time_total_load $file_time_parse $file_time_load;
    
$file_time_total_parse $file_time_end $file_time_parse;
    
$file_time_total $file_time_end $file_time_start;
    echo 
"Result: " $Data[count($Data)-1][0] . "<br>\n";
    echo 
"Open Time: " $file_time_total_open " seconds<br>\n";
    echo 
"Load Time: " $file_time_total_load " seconds<br>\n";
    echo 
"Parse Time: " $file_time_total_parse " seconds<br>\n";
    echo 
"Total Time: " $file_time_total " seconds<br>\n";


}

function 
FileLoad() {

    
$file_dir "";    // needs / at the end if anything other than the current directory
    
$file_name "spells_us.txt";
    
    
$file_time_start microtime(true);
    if (
file_exists($file_dir $file_name) && is_readable($file_dir $file_name)) {

        
$file_time_load microtime(true);
        
$Lines file($file_dir $file_name);
        
        
$file_time_parse microtime(true);
        foreach(
$Lines as $key => $value) {
            
$Data[$key] = explode("^"$value);
        }

        
$file_time_end microtime(true);

    } else {
        echo 
"Unable to read: " $file_dir $file_name "<br>\n";
    }
    
    if (!
$file_time_end)
        
$file_time_end microtime(true);
    
    
$file_time_total_open $file_time_load $file_time_start;
    
$file_time_total_load $file_time_parse $file_time_load;
    
$file_time_total_parse $file_time_end $file_time_parse;
    
$file_time_total $file_time_end $file_time_start;
    echo 
"Spells Loaded: " count($Lines) . "<br>\n";
    echo 
"Open Time: " $file_time_total_open " seconds<br>\n";
    echo 
"Load Time: " $file_time_total_load " seconds<br>\n";
    echo 
"Parse Time: " $file_time_total_parse " seconds<br>\n";
    echo 
"Total Time: " $file_time_total " seconds<br>\n";
    
}

echo 
"<b>DatabaseGetMax()</b><br>\n";
DatabaseGetMax();
echo 
"<br>\n";
echo 
"<b>DatabaseLoad()</b><br>\n";
DatabaseLoad();
echo 
"<br>\n";
echo 
"<b>FileGetMax()</b><br>\n";
FileGetMax();
echo 
"<br>\n";
echo 
"<b>FileLoad()</b><br>\n";
FileLoad();

?>

</html>
Code:
DatabaseGetMax()
Result: 11999
Class Time: 0.000842094421387 seconds
Query Time: 0.000253915786743 seconds (MySQL Time: 0.000201940536499 seconds)
Total Time: 0.00109601020813 seconds

DatabaseLoad()
Spells Loaded: 11540
Class Time: 0.000181913375854 seconds
Query Time: 1.7691822052 seconds (MySQL Time: 0.547831058502 seconds)
Total Time: 1.76936411858 seconds

FileGetMax()
Result: 11999
Open Time: 3.50475311279E-05 seconds
Load Time: 0.0381419658661 seconds
Parse Time: 0.978446960449 seconds
Total Time: 1.01662397385 seconds

FileLoad()
Spells Loaded: 11540
Open Time: 3.38554382324E-05 seconds
Load Time: 0.0379700660706 seconds
Parse Time: 0.978180885315 seconds
Total Time: 1.01618480682 seconds
However, if we are talking about JUST loading the spells (which can't be done, since we need to know what the highest spell ID is before we load them into shared memory), loading from a file is considerably faster.

Quote:
Originally Posted by KLS View Post
-In my opinion you made it harder on new users who now have to try to figure out 'hey there's two systems for spells, wait do I need them both? I can't find the db table I need anywhere help!!'.
That's why I'm not sure if it would be better to give the option or to shove it down everyone's throat (I prefer the former). I guess the expectation would be to switch it over to completely DB. Giving the options would just give an "easier" transition in the short term.

Quote:
Originally Posted by MNWatchdog View Post
It wasnt broke, so why are you fixing it?

Youre just adding an additional level of complication IMO.
Why have a database if we're not going to take advantage of it? Yes, we can have the spells stored in a file, but we could also have items, spawns, NPC types, merchant lists, spell lists, AAs, zones, zone servers, etc all in files. But that's just crazy. That's why we have a variables system & a rules system instead of config files like with the MaNGOS emulator (for WoW).
__________________
GM-Impossible of 'A work in progress'
A non-legit PEQ DB server
How to create your own non-legit server

My Contributions to the Wiki
Reply With Quote
  #2  
Old 10-28-2008, 03:28 AM
KLS
Administrator
 
Join Date: Sep 2006
Posts: 1,348
Default

I think it's a good idea as a supplement to tools but again I'm not sure it's the greatest idea for the base server. I don't want to discourage you if you really want to put forth the effort to not only create the database but create the tools to input and extract spells from it.

However I don't see much reason to actually put it in the server standalone is what I was trying to say, having a unified standard for tools would simplify their design considerably.
Reply With Quote
  #3  
Old 10-28-2008, 03:34 AM
MNWatchdog
Hill Giant
 
Join Date: Feb 2006
Posts: 179
Default

Theres a millions ways to trap a mouse, but the snap trap works with 95% efficiency.

Personally, I think you should change to the DB method and it only with a importer and exporter app for the spells.txt file.
Reply With Quote
  #4  
Old 10-28-2008, 05:15 AM
AndMetal
Developer
 
Join Date: Mar 2007
Location: Ohio
Posts: 648
Default

Quote:
Originally Posted by MNWatchdog View Post
Personally, I think you should change to the DB method and it only with a importer and exporter app for the spells.txt file.
I think that's the route I'll go.

I'll probably just make a Perl script to do both (since not everyone has PHP installed on their server systems), although I'm already to the point where I have about 80% of the base Titanium spell file in a .sql file that can be easily sourced (80% being the table structure, 20% importing it from the spells file).
__________________
GM-Impossible of 'A work in progress'
A non-legit PEQ DB server
How to create your own non-legit server

My Contributions to the Wiki
Reply With Quote
  #5  
Old 11-18-2008, 08:00 AM
AndMetal
Developer
 
Join Date: Mar 2007
Location: Ohio
Posts: 648
Default

Quote:
Originally Posted by AndMetal View Post
I'll probably just make a Perl script
http://code.google.com/p/projecteqem...port_spells.pl

Code:
Usage: import_spells.pl [-c path] [-s path] [-t table] [-d]
  -c path       path/to/eqemu_config.xml. defaults to eqemu_config.xml
  -s path       path/to/spells_us.txt. defaults to spells_us.txt
  -t table      table to load the spells into. defaults to spells_new
  -d            erase all spells from the database first
Tested it out several times and it works fine, including the erasing of the table.

I feel there are probably some opportunities to clean up the script a little (I've never worked with DBI before), so if anyone has any recommendations, let me know (or if you have SVN access, feel free to change it).
__________________
GM-Impossible of 'A work in progress'
A non-legit PEQ DB server
How to create your own non-legit server

My Contributions to the Wiki
Reply With Quote
  #6  
Old 11-24-2008, 07:13 AM
AndMetal
Developer
 
Join Date: Mar 2007
Location: Ohio
Posts: 648
Default

Just finished up the export script:

Code:
Usage: import_spells.pl [-c path] [-t table] [-i column] [-s path] [-o]
  -c path       path/to/eqemu_config.xml. defaults to eqemu_config.xml
  -t table      table to load the spells from. defaults to spells_new
  -i column     name of the column in the database table to order by. defaults to id
  -s path       path/to/export/spells_us.txt. defaults to spells_us.txt
  -o            overwrite spells_us.txt if it exists
The is also now officially in SVN as of revision 230. If anyone is confused about what to do, let me know.
__________________
GM-Impossible of 'A work in progress'
A non-legit PEQ DB server
How to create your own non-legit server

My Contributions to the Wiki
Reply With Quote
  #7  
Old 11-24-2008, 08:34 AM
trevius's Avatar
trevius
Developer
 
Join Date: Aug 2006
Location: USA
Posts: 5,946
Default

Can you give an example of using the script? Also, how do you disable using this so it continues to use the normal spells file? From reading the changelog, it sounds like it is supposed to use this table by default. I am guessing that means it will ignore the spells file unless you disable this feature? Or, will it use the spells file if it can't find the table?

BTW, your code box in your export script post says "import", not "export" for usage.

So, to import spells into the database, I would do this?:

Code:
/utils/import_spells.pl -c /home/eqemu/server/eqemu_config.xml -s /home/eqemu/server/spells_us.txt -t spells_new -d
And to export the table, I would do this?:

Code:
/utils/export_spells.pl -c /home/eqemu/server/eqemu_config.xml -t spells_new -s /home/eqemu/server/spells_us.txt
EDIT: Just tried to run this, but I haven't ran a .pl file before and it doesn't seem to run just from a terminal window. Is there something else I need to do to run it?

Code:
eqemu@muse:~/source/EQEmuServer2/utils$ ls 
0.6.1-upgrade.sql  items-0.6.0-DR2-0.6.1-DR1-convert.sql  ppskillfix
apathing           itemtablechanges.sql                   schema.xml
asmtools           load_13thfloor_items.pl                serialize_items.pl
azone              Makefile                               spell_explorer.cpp
azone2             opcodes.conf                           sql
charmove           patch_6.2.conf                         struct_fields.sh
cleanipc.cpp       patch_Anniversary.conf                 TaskMaster
defaults           patch_Live.conf                        throwpackets.pl
export_spells.pl   patch_Titanium.conf                    wr_update.sql
hexvis             perlxs                                 ZONECFG.SQL
import_showeq      ppconvert
import_spells.pl   ppreader.pl
eqemu@muse:~/source/EQEmuServer2/utils$ import_spells.pl -h bash: import_spells.pl: command not found
eqemu@muse:~/source/EQEmuServer2/utils$ ./import_spells.pl -h
bash: ./import_spells.pl: /usr/bin/perl^M: bad interpreter: No such file or directory
eqemu@muse:~/source/EQEmuServer2/utils$
__________________
Trevazar/Trevius Owner of: Storm Haven
Everquest Emulator FAQ (Frequently Asked Questions) - Read It!

Last edited by trevius; 11-24-2008 at 04:51 PM..
Reply With Quote
  #8  
Old 11-24-2008, 08:55 AM
jenco420
Banned
 
Join Date: Jan 2006
Location: /dev/null
Posts: 99
Default

should be able to just

Code:
sh export_spells.pl
but i don't have a linux box near me atm
Reply With Quote
  #9  
Old 11-24-2008, 09:22 AM
trevius's Avatar
trevius
Developer
 
Join Date: Aug 2006
Location: USA
Posts: 5,946
Default

This is what I get from sh:

Code:
eqemu@muse:~/source/EQEmuServer2/utils$ sh export_spells.pl -h
: command not foundine 2:
export_spells.pl: line 5: use: command not found
: command not foundine 5:
export_spells.pl: line 6: use: command not found
: command not foundine 6:
: command not foundine 7:
export_spells.pl: line 9: syntax error near unexpected token `'c:t:i:s:oh''
'xport_spells.pl: line 9: `getopts('c:t:i:s:oh');
eqemu@muse:~/source/EQEmuServer2/utils$ sh import_spells.pl -c /home/eqemu/server/eqemu_config.xml -s /home/eqemu/server/spells_us.txt -t spells_new -d
: command not foundine 2:
import_spells.pl: line 5: use: command not found
: command not foundine 5:
import_spells.pl: line 6: use: command not found
: command not foundine 6:
: command not foundine 7:
import_spells.pl: line 9: syntax error near unexpected token `'c:s:t:dh''
'mport_spells.pl: line 9: `getopts('c:s:t:dh');
eqemu@muse:~/source/EQEmuServer2/utils$
And, to set it back to load the normal spell file, I am guessing I need to modify this part of the spdat.h?:

Code:
//#define NEW_LoadSPDat
#define DB_LoadSPDat	//load from DB vs spells_us.txt. for now, we're piggybacking NEW_LoadSPDat, so it will take precedence
Or does that mean this is still disabled by default? Change log says enabled by default, so I am not understanding. Do I just need to comment out #define DB_LoadSPDat ? I can't even get it to compile if that line is commented out. If loading spells from the DB is now enabled by default, and it doesn't automatically use the spells_us.txt file if the spells_new table isn't available, then I think this needs to be disabled ASAP. Or we at least need a complete noobie guide to setting it up, because I know that if I am having trouble with it, many others will too.
__________________
Trevazar/Trevius Owner of: Storm Haven
Everquest Emulator FAQ (Frequently Asked Questions) - Read It!

Last edited by trevius; 11-24-2008 at 05:39 PM..
Reply With Quote
Reply

Thread Tools
Display Modes

Posting Rules
You may not post new threads
You may not post replies
You may not post attachments
You may not edit your posts

BB code is On
Smilies are On
[IMG] code is On
HTML code is Off

Forum Jump

   

All times are GMT -4. The time now is 07:47 AM.


 

Everquest is a registered trademark of Daybreak Game Company LLC.
EQEmulator is not associated or affiliated in any way with Daybreak Game Company LLC.
Except where otherwise noted, this site is licensed under a Creative Commons License.
       
Powered by vBulletin®, Copyright ©2000 - 2025, Jelsoft Enterprises Ltd.
Template by Bluepearl Design and vBulletin Templates - Ver3.3