coreseek安装配置以及,索引与增量索引配置

wget https://www.isres.com/file/coreseek-3.2.14.tar.gz
tar -zxvf coreseek-3.2.14.tar.gz
cd coreseek-3.2.14
cd mmseg-3.2.14
./configure --prefix=/usr/local/mmseg
#报错
config.status: error: cannot find input file: src/Makefile.in
aclocal
libtoolize --force
automake --add-missing
autoconf 
autoheader 
make clean
#-------------------------------------------
./configure --prefix=/usr/local/mmseg
make && make install

#报错
/bin/sh: ./config.status: No such file or directory
make: *** [Makefile] Error 127
[root@shw-xuniji mmseg-3.2.14]# ./config.status Makefile
#-------------------------------------------

cd ../
cd csft-3.2.14/
./configure --prefix=/usr/local/coreseek --with-mmseg=/usr/local/mmseg --with-mmseg-includes=/usr/local/mmseg/include/mmseg/ --with-mmseg-libs=/usr/local/mmseg/lib/
make && make install

编辑sphinx.conf


source src1
{
        type                                    = mysql

        sql_host                                = 127.0.0.1
        sql_user                                = root
        sql_pass                                = aaaaa
        sql_db                                  = shwcms_v10
        sql_port                                = 3306  # optional, default is 3306

        sql_query                               = \
                SELECT id,fenlei,biaoti \
                FROM shwcms_xinwenwenzhang


        sql_query_pre = SET NAMES utf8

        sql_query_info                  = SELECT * FROM shwcms_xinwenwenzhang WHERE id=$id
        sql_attr_uint    = id

}


index index1
{
        source                                  = src1
        path                                    = /usr/local/coreseek/var/data/src1
        docinfo                                 = extern
        charset_dictpath = /usr/local/mmseg/etc/
        ngram_len = 0

        charset_type                    = zh_cn.utf-8
}


#############################################################################
## indexer settings
#############################################################################

indexer
{
    # memory limit, in bytes, kiloytes (16384K) or megabytes (256M)
    # optional, default is 32M, max is 2047M, recommended is 256M to 1024M
    #mem_limit            = 1024M

    # maximum IO calls per second (for I/O throttling)
    # optional, default is 0 (unlimited)
    #
    #max_iops            = 400


    # maximum IO call size, bytes (for I/O throttling)
    # optional, default is 0 (unlimited)
    #
    #max_iosize        = 5048576


    # maximum xmlpipe2 field length, bytes
    # optional, default is 2M
    #
    max_xmlpipe2_field    = 4M


    # write buffer size, bytes
    # several (currently up to 4) buffers will be allocated
    # write buffers are allocated in addition to mem_limit
    # optional, default is 1M
    #
    # write_buffer        = 1M
}

#############################################################################
## searchd settings
#############################################################################

searchd
{
        port                                    = 9312
        log                                             = /usr/local/coreseek/var/log/searchd.log
        query_log                               = /usr/local/coreseek/var/log/query.log
        read_timeout                    = 5
        max_children                    = 1000
        pid_file                                = /usr/local/coreseek/var/log/searchd.pid
        max_matches                             = 2000
        seamless_rotate                 = 1
        preopen_indexes                 = 0
        unlink_old                              = 1
}

/usr/local/coreseek/bin/indexer -c /usr/local/coreseek/etc/sphinx.conf --all
/usr/local/coreseek/bin/searchd -c  /usr/local/coreseek/etc/sphinx.conf
增量,非实时 只是不需要全部重建

coreseek配置文件sphinx.conf

#
# Minimal Sphinx configuration sample (clean, simple, functional)
#

#门窗源
source menwang{
    type                    = mysql

        sql_host                = 192.168.1.2
        sql_user                = user
        sql_pass                = pwd
        sql_db                  = dbname
        sql_port                = 3306  # optional, default is 3306
        sql_query_pre = replace into dt_sphinx select 1,max(itemid) from dt_article_21
        sql_query               = \
                SELECT itemid,title,username from dt_article_21 where itemid <=(select maxid from dt_sphinx where moduleid = 1)
    sql_query_pre = SET NAMES utf8
    #                         
}
#门窗增量源
source delta_menwang:menwang{
    sql_query_pre = SET NAMES utf8
        sql_query = SELECT itemid,title,username from dt_article_21 where itemid>( select maxid from dt_sphinx where moduleid = 1 )    
}

#门窗索引
index menwang{
    source                  = menwang
        path                    = /usr/local/coreseek/var/data/menwang
        docinfo                 = extern
        stopwords = /usr/local/coreseek/var/data/stopwords.txt
        charset_dictpath = /usr/local/mmseg3/etc/
        ngram_len = 0

        charset_type                    = zh_cn.utf-8

}
#门窗增量
index delta_menwang:menwang
{
    source            = delta_menwang
    path            = /usr/local/coreseek/var/data/delta_menwang
    docinfo                 = extern
    stopwords = /usr/local/coreseek/var/data/stopwords.txt
        charset_dictpath = /usr/local/mmseg3/etc/
        ngram_len = 0

        charset_type                    = zh_cn.utf-8
}




indexer
{
    mem_limit        = 128M
}


searchd
{
    listen            = 9312
    listen            = 9306:mysql41
    log            = /usr/local/coreseek/var/log/searchd.log
    query_log        = /usr/local/coreseek/var/log/query.log
    read_timeout        = 5
    max_children        = 30
    pid_file        = /usr/local/coreseek/var/log/searchd.pid
    max_matches        = 1000
    seamless_rotate        = 1
    preopen_indexes        = 1
    unlink_old        = 1
    workers            = threads # for RT to work
}

shell重建索引脚本

#!/bin/sh 
/usr/local/coreseek/bin/searchd --stop -c /usr/local/coreseek/etc/sphinx.conf >> /shell/`date "+%Y-%m-%d %H:%M:%S"`_stop.log
sleep 2
/usr/local/coreseek/bin/indexer -c /usr/local/coreseek/etc/sphinx.conf --all >> /shell/`date "+%Y-%m-%d %H:%M:%S"`_index.log
sleep 2
/usr/local/coreseek/bin/searchd -c  /usr/local/coreseek/etc/sphinx.conf >> /shell/`date "+%Y-%m-%d %H:%M:%S"`_start.log

shell增量索引脚本

#!/bin/sh  
/usr/local/coreseek/bin/indexer -c /usr/local/coreseek/etc/sphinx.conf delta_menwang --rotate >> /shell/sp_menwang_zengliang.log
sleep 1
/usr/local/coreseek/bin/indexer -c /usr/local/coreseek/etc/sphinx.conf --merge menwang delta_menwang --rotate --merge-dst-range deleted 0 0 >> /shell/sp_menwang_zengliang.log
echo `date "+%Y-%m-%d %H:%M:%S"` >> /shell/sp_menwang_zengliang.log

crontab 定时任务增量

*/1 * * * * /bin/sh /shell/sp_menwang.zengliang.sh

增量涉及的数据表

CREATE TABLE `sph_counter` (
  `counter_id` int(4) NOT NULL,
  `max_doc_id` bigint(20) DEFAULT NULL,
  PRIMARY KEY (`counter_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

要定期重建索引,比如每天0点。
增量根据业务场景,最快可每分钟增量一次。
增量索引不会影响 原已索引历史数据中的数据,即 如果在真实表上修改了标题,必须以重建索引的方式做到更新。


标签: coreseek

非特殊说明,本博所有文章均为博主原创。

最新文章

发表评论