Поисковые боты

Dream

Пользователь
ты смотри по коду и файлы какие мы редактируем, где что нужно находить и заменять...
и всё будет ясно
 
М

мегалайн

Гость
Запрос в БД:
Код:
ALTER TABLE `bb_sessions` ADD `bot_name` VARCHAR( 255 ) NOT NULL ;
Откроем /includes/online_userlist.php и ищем:
Код:
$logged_online = $guests_online = 0;
$time_online = TIMENOW - 300;
#    $time_online = 0;
 
$ulist = array(
    ADMIN        => array(),
    MOD          => array(),
    GROUP_MEMBER => array(),
    USER        => array(),
);
$users_cnt = array(
    'admin'        => 0,
    'mod'          => 0,
    'group_member' => 0,
    'ignore_load'  => 0,
    'user'        => 0,
    'guest'        => 0,
);
$online = $online_short = array('userlist' => '');
 
$sql = "
    SELECT
        u.username, u.user_id, u.user_opt, u.user_rank, u.user_level,
        u.ignore_srv_load,
        s.session_logged_in, s.session_ip, (s.session_time - s.session_start) AS ses_len, COUNT(s.session_id) AS sessions, COUNT(DISTINCT s.session_ip) AS ips
    FROM ". BB_SESSIONS ." s, ". BB_USERS ." u
    WHERE s.session_time > $time_online
        AND u.user_id = s.session_user_id
    GROUP BY s.session_user_id
    ORDER BY u.username
";
 
foreach (DB()->fetch_rowset($sql) as $u)
{
    if ($u['session_logged_in'])
    {
        $stat = array();
        $name = profile_url($u);
        $level = $u['user_level'];
 
        if ($level == ADMIN)
        {
            $name = "<b>$name</b>";
            $users_cnt['admin']++;
        }
        else if ($level == MOD)
        {
            $name = "<b>$name</b>";
            $users_cnt['mod']++;
        }
        else if ($level == GROUP_MEMBER)
        {
            $name = "<b>$name</b>";
            $users_cnt['group_member']++;
        }
        else
        {
            $users_cnt['user']++;
        }
 
        if ($u['sessions'] > 3)
        {
            $color = ($u['sessions'] > 2) ? '#FF0000' : '#B22222';
            $s = $u['sessions'];
            $stat[] = "s:<span style=\"color: $color\">$s</span>";
        }
        if ($u['ips'] > 2)
        {
            $ip = $u['ips'];
            $stat[] = "ip:<span style=\"color: #0000FF\">$ip</span>";
        }
        if ($u['ses_len'] > 6*3600 && $level == USER)
        {
            $t = round($u['ses_len'] / 3600, 1);
            $stat[] = "t:<span style=\"color: #1E90FF\">$t</span>";
        }
 
        $ulist[$level][] = ($stat) ? "$name<span class=\"ou_stat\" style=\"color: #707070\" title=\"{$u['session_ip']}\"> [<b>". join(', ', $stat) .'</b>]</span>' : $name;
    }
    else
    {
        $guests_online = $u['ips'];
        $users_cnt['guest'] = $guests_online;
    }
}
 
if ($ulist)
{
    $inline = $block = $short = array();
 
    foreach ($ulist as $level => $users)
    {
        if (empty($users)) continue;
 
        if (count($users) > 200)
        {
            $style = 'margin: 3px 0; padding: 2px 4px; border: 1px inset; height: 200px; overflow: auto;';
            $block[] = "<div style=\"$style\">\n". join(",\n", $users) ."</div>\n";
            $short[] = '<a href="index.php?online_full=1#online">'. $lang['USERS'] .': '. count($users) .'</a>';
        }
        else
        {
            $inline[] = join(",\n", $users);
            $short[]  = join(",\n", $users);
        }
 
        $logged_online += count($users);
    }
 
    $online['userlist'] = join(",\n", $inline) . join("\n", $block);
    $online_short['userlist'] = join(",\n", $short);
}
 
if (!$online['userlist'])
{
    $online['userlist'] = $online_short['userlist'] = $lang['NONE'];
}
else if (isset($_REQUEST['f']))
{
    $online['userlist'] = $online_short['userlist'] = $lang['BROWSING_FORUM'] .' '. $online['userlist'];
}
 
$total_online = $logged_online + $guests_online;
 
if ($total_online > $bb_cfg['record_online_users'])
{
    bb_update_config(array(
        'record_online_users' => $total_online,
        'record_online_date'  => TIMENOW,
    ));
}
 
$online['stat'] = $online_short['stat'] = sprintf($lang['ONLINE_USERS'], $total_online, $logged_online, $guests_online);
Заменить на это:
Код:
$logged_online = $guests_online = $bots_online = $prev_user = 0;
$time_online = TIMENOW - 300;
$bot = array();
 
$ulist = array(
    ADMIN        => array(),
    MOD          => array(),
    GROUP_MEMBER => array(),
    USER        => array(),
);
$users_cnt = array(
    'admin'        => 0,
    'mod'          => 0,
    'group_member' => 0,
    'ignore_load'  => 0,
    'user'        => 0,
    'guest'        => 0,
);
$online = $online_short = array('userlist' => '');
 
$sql = "
    SELECT
        u.username, u.user_id, u.user_opt, u.user_rank, u.user_level,
        u.ignore_srv_load,
        s.session_logged_in, s.bot_name, s.session_ip, (s.session_time - s.session_start) AS ses_len,
        COUNT(s.session_id) AS sessions,
        COUNT(DISTINCT s.session_ip) AS ips
    FROM ". BB_SESSIONS ." s, ". BB_USERS ." u
    WHERE s.session_time > $time_online
        AND u.user_id = s.session_user_id
    GROUP BY s.session_id
    ORDER BY u.username
";
 
foreach (DB()->fetch_rowset($sql) as $u)
{
    if ($u['session_logged_in'])
    {
        if ( $u['user_id'] != $prev_user )
        {
            $stat = array();
            $name = profile_url($u);
            $level = $u['user_level'];
 
            if ($level == ADMIN)
            {
                $name = "<b>$name</b>";
                $users_cnt['admin']++;
            }
            else if ($level == MOD)
            {
                $name = "<b>$name</b>";
                $users_cnt['mod']++;
            }
            else if ($level == GROUP_MEMBER)
            {
                $name = "<b>$name</b>";
                $users_cnt['group_member']++;
            }
            else
            {
                $users_cnt['user']++;
            }
 
            if ($u['sessions'] > 3)
            {
                $color = ($u['sessions'] > 2) ? '#FF0000' : '#B22222';
                $s = $u['sessions'];
                $stat[] = "s:<span style=\"color: $color\">$s</span>";
            }
            if ($u['ips'] > 2)
            {
                $ip = $u['ips'];
                $stat[] = "ip:<span style=\"color: #0000FF\">$ip</span>";
            }
            if ($u['ses_len'] > 6*3600 && $level == USER)
            {
                $t = round($u['ses_len'] / 3600, 1);
                $stat[] = "t:<span style=\"color: #1E90FF\">$t</span>";
            }
 
            $ulist[$level][] = ($stat) ? "$name<span class=\"ou_stat\" style=\"color: #707070\" title=\"". decode_ip($u['session_ip']) ."\"> [<b>". join(', ', $stat) .'</b>]</span>' : $name;
            $prev_user = $u['user_id'];
        }
    }
    elseif($u['bot_name'])
    {
        $bots_online++;
        if(isset($ulist[ANONYMOUS][$u['bot_name']]))
        {
            $bot[$u['bot_name']]++;
            $ulist[ANONYMOUS][$u['bot_name']] = $u['bot_name'] .' ['. ($bot[$u['bot_name']]) .']';
        }
        else
        {
            $bot[$u['bot_name']] = 1;
            $ulist[ANONYMOUS][$u['bot_name']] = $u['bot_name'];
        }
    }
    else
    {
        $guests_online++;
        $users_cnt['guest'] = $guests_online;
    }
}
 
if ($ulist)
{
    $inline = $block = $short = array();
 
    foreach ($ulist as $level => $users)
    {
        if (empty($users)) continue;
 
        if (count($users) > 200)
        {
            $style = 'margin: 3px 0; padding: 2px 4px; border: 1px inset; height: 200px; overflow: auto;';
            $block[] = "<div style=\"$style\">\n". join(",\n", $users) ."</div>\n";
            $short[] = '<a href="index.php?online_full=1#online">'. $lang['USERS'] .': '. count($users) .'</a>';
        }
        else
        {
            $inline[] = join(",\n", $users);
            $short[]  = join(",\n", $users);
        }
 
        if($level != ANONYMOUS) $logged_online += count($users);
    }
 
    $online['userlist'] = join(",\n", $inline) . join("\n", $block);
    $online_short['userlist'] = join(",\n", $short);
}
 
if (!$online['userlist'])
{
    $online['userlist'] = $online_short['userlist'] = $lang['NONE'];
}
else if (isset($_REQUEST['f']))
{
    $online['userlist'] = $online_short['userlist'] = $lang['BROWSING_FORUM'] .' '. $online['userlist'];
}
 
$total_online = $logged_online + $guests_online + $bots_online;
 
if ($total_online > $bb_cfg['record_online_users'])
{
    bb_update_config(array(
        'record_online_users' => $total_online,
        'record_online_date'  => TIMENOW,
    ));
}
 
$online['stat'] = $online_short['stat'] = sprintf($lang['ONLINE_USERS'], $total_online, $logged_online, $guests_online, $bots_online);
Откроем /includes/sessions.php и ищем:
Код:
function session_start ($cfg = array())
    {
После
Код:
global $bb_cfg;
Вставим:
Код:
$user_agent = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : 'not browser';
После
Код:
$this->cfg = array_merge($this->cfg, $cfg);
Встаим:
Код:
$bot_name = '';
        foreach ($bb_cfg['bots'] as $bot => $name)
        {
            if(strstr($user_agent, $bot)) $bot_name = $name;
        }
Ищем:
Код:
// Only update session a minute or so after last update
                if ($update_sessions_table)
                {
                    DB()->query("
                        UPDATE ". BB_SESSIONS ." SET
                            session_time = ". TIMENOW ."
                        WHERE session_id = '$session_id'
                        LIMIT 1
                    ");
                }
и заменить на
Код:
// Only update session a minute or so after last update
                if ($update_sessions_table)
                {
                    DB()->query("
                        UPDATE ". BB_SESSIONS ." SET
                            session_time = ". TIMENOW .",
                            bot_name = '$bot_name'
                        WHERE session_id = '$session_id'
                        LIMIT 1
                    ");
                }
Ищем:
Код:
function session_create ($userdata, $auto_created = false)
{
После
Код:
global $bb_cfg;
Встаить:
Код:
$user_browser = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : 'not browser';
После
Код:
$mod_admin_session = ($this->data['user_level'] == ADMIN || $this->data['user_level'] == MOD);
Вставить:
Код:
$bot_name = '';
        foreach ($bb_cfg['bots'] as $bot => $name)
        {
            if(strstr($user_browser, $bot)) $bot_name = $name;
        }
Ищем:
Код:
$args = DB()->build_array('INSERT', array(
                'session_id'        => (string) $session_id,
                'session_user_id'  => (int) $user_id,
                'session_start'    => (int) TIMENOW,
                'session_time'      => (int) TIMENOW,
                'session_ip'        => (string) USER_IP,
                'session_logged_in' => (int) $login,
                'session_admin'    => (int) $mod_admin_session,
            ));
            $sql = "INSERT INTO ". BB_SESSIONS . $args;
Заменить на:
Код:
$args = DB()->build_array('INSERT', array(
                'session_id'        => (string) $session_id,
                'session_user_id'  => (int) $user_id,
                'session_start'    => (int) TIMENOW,
                'session_time'      => (int) TIMENOW,
                'session_ip'        => (string) USER_IP,
                'session_logged_in' => (int) $login,
                'session_admin'    => (int) $mod_admin_session,
                'bot_name'          => $bot_name,
            ));
            $sql = "INSERT INTO ". BB_SESSIONS . $args;
Откроем config.php и в конец добавим:
Код:
$bb_cfg['bots'] = array (
    'AdsBot-Google' => 'AdsBot [Google]',
    'ia_archiver' => 'Alexa [Bot]',
    'Scooter/' => 'Alta Vista [Bot]',
    'Ask Jeeves' => 'Ask Jeeves [Bot]',
    'Baiduspider+(' => 'Baidu [Spider]',
    'Exabot/' => 'Exabot [Bot]',
    'FAST Enterprise Crawler' => 'FAST Enterprise [Crawler]',
    'FAST-WebCrawler/' => 'FAST WebCrawler [Crawler]',
    'http://www.neomo.de/' => 'Francis [Bot]',
    'Gigabot/' => 'Gigabot [Bot]',
    'Mediapartners-Google' => 'Google Adsense [Bot]',
    'Google Desktop' => 'Google Desktop',
    'Feedfetcher-Google' => 'Google Feedfetcher',
    'Googlebot' => 'Google [Bot]',
    'heise-IT-Markt-Crawler' => 'Heise IT-Markt [Crawler]',
    'heritrix/1.' => 'Heritrix [Crawler]',
    'ibm.com/cs/crawler' => 'IBM Research [Bot]',
    'ICCrawler - ICjobs' => 'ICCrawler - ICjobs',
    'ichiro/' => 'ichiro [Crawler]',
    'MJ12bot/' => 'Majestic-12 [Bot]',
    'MetagerBot/' => 'Metager [Bot]',
    'msnbot-NewsBlogs/' => 'MSN NewsBlogs',
    'msnbot/' => 'MSN [Bot]',
    'msnbot-media/' => 'MSNbot Media',
    'NG-Search/' => 'NG-Search [Bot]',
    'http://lucene.apache.org/nutch/' => 'Nutch [Bot]',
    'NutchCVS/' => 'Nutch/CVS [Bot]',
    'OmniExplorer_Bot/' => 'OmniExplorer [Bot]',
    'online link validator' => 'Online link [Validator]',
    'psbot/0' => 'psbot [Picsearch]',
    'Seekbot/' => 'Seekport [Bot]',
    'Sensis Web Crawler' => 'Sensis [Crawler]',
    'SEO search Crawler/' => 'SEO Crawler',
    'Seoma [SEO Crawler]' => 'Seoma [Crawler]',
    'SEOsearch/' => 'SEOSearch [Crawler]',
    'Snappy/1.1 ( http://www.urltrends.com/ )' => 'Snappy [Bot]',
    'http://www.tkl.iis.u-tokyo.ac.jp/~crawler/' => 'Steeler [Crawler]',
    'SynooBot/' => 'Synoo [Bot]',
    '[email protected]' => 'Telekom [Bot]',
    'TurnitinBot/' => 'TurnitinBot [Bot]',
    'voyager/1.0' => 'Voyager [Bot]',
    'W3 SiteSearch Crawler' => 'W3 [Sitesearch]',
    'W3C-checklink/' => 'W3C [Linkcheck]',
    'W3C_*Validator' => 'W3C [Validator]',
    'http://www.WISEnutbot.com' => 'WiseNut [Bot]',
    'yacybot' => 'YaCy [Bot]',
    'Yahoo-MMCrawler/' => 'Yahoo MMCrawler [Bot]',
    'Yahoo! DE Slurp' => 'Yahoo Slurp [Bot]',
    'Yahoo! Slurp' => 'Yahoo [Bot]',
    'YahooSeeker/' => 'YahooSeeker [Bot]',
    'Yandex/1.01.001 (compatible; Win16; I)' => 'Яндекс БОТ',
    'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)' => 'Яндекс [Bot]',
    'bingbot/' => 'Bing [Bot]',
    'Rambler/' => 'Rambler [Bot]',
);
Откроем viewonline.php и ищем:
Код:
$sql = "SELECT u.user_id, u.username, u.user_opt, u.user_rank,u.user_avatar, u.user_avatar_type, s.session_logged_in, s.session_time, s.session_ip
    FROM ".BB_USERS." u, ".BB_SESSIONS." s
    WHERE u.user_id = s.session_user_id
        AND s.session_time >= ".( TIMENOW - 300 ) . "
    ORDER BY u.username ASC, s.session_ip ASC";
if ( !($result = DB()->sql_query($sql)) )
{
    message_die(GENERAL_ERROR, 'Could not obtain regd user/online information', '', __LINE__, __FILE__, $sql);
}
 
$guest_users = 0;
$registered_users = 0;
$hidden_users = 0;
 
$reg_counter = 0;
$guest_counter = 0;
$prev_user = 0;
$prev_ip = '';
 
$user_id = 0;
 
while ( $row = DB()->sql_fetchrow($result) )
{
    $view_online = false;
 
    if ( $row['session_logged_in'] )
    {
        $user_id = $row['user_id'];
 
        if ( $user_id != $prev_user )
        {
            $username = profile_url($row);
 
            if ( bf($row['user_opt'], 'user_opt', 'allow_viewonline') )
            {
                $view_online = IS_AM;
                $hidden_users++;
 
                $username = '<i>' . $username . '</i>';
            }
            else
            {
                $view_online = true;
                $registered_users++;
            }
 
            $which_counter = 'reg_counter';
            $which_row = 'reg_user_row';
            $prev_user = $user_id;
        }
    }
    else
    {
        if ( $row['session_ip'] != $prev_ip )
        {
            $username = $lang['GUEST'];
            $view_online = true;
            $guest_users++;
 
            $which_counter = 'guest_counter';
            $which_row = 'guest_user_row';
        }
    }
 
    $prev_ip = $row['session_ip'];
    $user_ip = hexdec(substr($prev_ip, 0, 2)) . '.' . hexdec(substr($prev_ip, 2, 2)) . '.' . hexdec(substr($prev_ip, 4, 2)) . '.' . hexdec(substr($prev_ip, 6, 2));
 
    if ( $view_online )
    {
        $row_class = !($which_counter % 2) ? 'row1' : 'row2';
 
        $template->assign_block_vars("$which_row", array(
            'ROW_CLASS'  => $row_class,
            'USER'      => $username,
            'AVATAR'    => get_avatar($row['user_avatar'], $row['user_avatar_type'], !bf($row['user_opt'], 'user_opt', 'allow_avatar'), 48 ,48),
            'LASTUPDATE' => bb_date($row['session_time']),
            'LASTUPDATE_RAW' => $row['session_time'],
            'USERIP'    => $user_ip,
            'U_WHOIS_IP' => $bb_cfg['whois_info'] . $user_ip,
        ));
 
        $which_counter++;
    }
}
 
$template->assign_vars(array(
    'TOTAL_USERS_ONLINE' => $registered_users + $hidden_users + $guest_users,
    'TOTAL_REGISTERED_USERS_ONLINE' => sprintf($lang['REG_USERS_ONLINE'], $registered_users) . sprintf($lang['HIDDEN_USERS_ONLINE'], $hidden_users),
    'TOTAL_GUEST_USERS_ONLINE' => sprintf($lang['GUEST_USERS_ONLINE'], $guest_users))
);
заменить на
Код:
$sql = "SELECT u.user_id, u.username, u.user_opt, u.user_rank, s.session_logged_in, s.session_time, s.session_ip, s.bot_name
    FROM ". BB_USERS ." u, ". BB_SESSIONS ." s
    WHERE u.user_id = s.session_user_id
        AND s.session_time >= ".( TIMENOW - 300 ) . "
    ORDER BY u.username ASC, s.session_ip ASC";
if ( !($result = DB()->sql_query($sql)) )
{
    message_die(GENERAL_ERROR, 'Could not obtain regd user/online information', '', __LINE__, __FILE__, $sql);
}
 
$guest_users = 0;
$registered_users = 0;
$hidden_users = 0;
$bots = 0;
 
$prev_user = 0;
$prev_ip = '';
 
$user_id = 0;
 
while ( $row = DB()->sql_fetchrow($result) )
{
    $view_online = false;
 
    if ( $row['session_logged_in'] )
    {
        $user_id = $row['user_id'];
 
        if ( $user_id != $prev_user )
        {
            $username = profile_url($row);
 
            if ( bf($row['user_opt'], 'user_opt', 'allow_viewonline') )
            {
                $view_online = IS_AM;
                $hidden_users++;
 
                $username = '<i>' . $username . '</i>';
            }
            else
            {
                $view_online = true;
                $registered_users++;
            }
 
            $which_counter = 'reg_counter';
            $which_row = 'reg_user_row';
            $prev_user = $user_id;
        }
    }
    else if($row['bot_name'])
    {
        $username = $row['bot_name'];
        $view_online = true;
        $bots++;
 
        $which_counter = 'bot_counter';
        $which_row = 'bot_user_row';
    }
    else
    {
        if ( $row['session_ip'] != $prev_ip )
        {
            $username = $lang['GUEST'];
            $view_online = true;
            $guest_users++;
 
            $which_counter = 'guest_counter';
            $which_row = 'guest_user_row';
        }
    }
 
    $prev_ip = $row['session_ip'];
    $user_ip = hexdec(substr($prev_ip, 0, 2)) . '.' . hexdec(substr($prev_ip, 2, 2)) . '.' . hexdec(substr($prev_ip, 4, 2)) . '.' . hexdec(substr($prev_ip, 6, 2));
 
    if ( $view_online )
    {
        $row_class = !($which_counter % 2) ? 'row1' : 'row2';
 
        $template->assign_block_vars("$which_row", array(
            'ROW_CLASS'  => $row_class,
            'USER'      => $username,
            'LASTUPDATE' => bb_date($row['session_time']),
            'LASTUPDATE_RAW' => $row['session_time'],
            'USERIP'    => $user_ip,
            'U_WHOIS_IP' => $bb_cfg['whois_info'] . $user_ip,
        ));
 
        $which_counter++;
    }
}
 
if( $registered_users == 0 )
{
    $l_r_user_s = $lang['REG_USERS_ZERO_ONLINE'];
}
else if( $registered_users == 1 )
{
    $l_r_user_s = $lang['REG_USER_ONLINE'];
}
else
{
    $l_r_user_s = $lang['REG_USERS_ONLINE'];
}
 
if( $hidden_users == 0 )
{
    $l_h_user_s = $lang['HIDDEN_USERS_ZERO_ONLINE'];
}
else if( $hidden_users == 1 )
{
    $l_h_user_s = $lang['HIDDEN_USER_ONLINE'];
}
else
{
    $l_h_user_s = $lang['HIDDEN_USERS_ONLINE'];
}
 
if( $guest_users == 0 )
{
    $l_g_user_s = $lang['GUEST_USERS_ZERO_ONLINE'];
}
else if( $guest_users == 1 )
{
    $l_g_user_s = $lang['GUEST_USER_ONLINE'];
}
else
{
    $l_g_user_s = $lang['GUEST_USERS_ONLINE'];
}
 
$template->assign_vars(array(
    'TOTAL_USERS_ONLINE' => $l_r_user_s + $registered_users + $l_h_user_s + $hidden_users + $l_g_user_s + $guest_users,
    'TOTAL_REGISTERED_USERS_ONLINE' => sprintf($l_r_user_s, $registered_users) . sprintf($l_h_user_s, $hidden_users),
    'TOTAL_GUEST_USERS_ONLINE' => sprintf($l_g_user_s, $guest_users),
    'TOTAL_BOTS_ONLINE' => $bots,
));
Открыть /templates/default/viewonline.tpl и после
Код:
<!-- END reg_user_row -->
вставить
Код:
<tr>
    <td class="catTitle" colspan="3">Сейчас на сайте поисковых ботов: {TOTAL_BOTS_ONLINE}</td>
</tr>
<!-- BEGIN bot_user_row -->
<tr class="{bot_user_row.ROW_CLASS}">
    <td><b>{bot_user_row.USER}</b></td>
    <td class="tCenter"><u>{bot_user_row.LASTUPDATE_RAW}</u>{bot_user_row.LASTUPDATE}</td>
    <!-- IF IS_ADMIN --><td class="tCenter"><a href="{bot_user_row.U_WHOIS_IP}" class="gen" target="_blank">{bot_user_row.USERIP}</a></td><!-- ENDIF -->
</tr>
<!-- END bot_user_row -->


Скажите это у меня правильно? По моему работает нормально, показывает ботов
 
Сверху