Initial commit

vendor/github.com/go-ego/riot/BUILD.bazel

@@ -0,0 +1,55 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "counters.go",
+        "engine.go",
+        "indexer_worker.go",
+        "info.go",
+        "ranker_worker.go",
+        "riot.go",
+        "riot_pkg.go",
+        "segment.go",
+        "stop_tokens.go",
+        "store_worker.go",
+    ],
+    importmap = "go-common/vendor/github.com/go-ego/riot",
+    importpath = "github.com/go-ego/riot",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/coreos/bbolt:go_default_library",
+        "//vendor/github.com/dgraph-io/badger:go_default_library",
+        "//vendor/github.com/go-ego/gpy:go_default_library",
+        "//vendor/github.com/go-ego/gse:go_default_library",
+        "//vendor/github.com/go-ego/murmur:go_default_library",
+        "//vendor/github.com/go-ego/riot/core:go_default_library",
+        "//vendor/github.com/go-ego/riot/store:go_default_library",
+        "//vendor/github.com/go-ego/riot/types:go_default_library",
+        "//vendor/github.com/go-ego/riot/utils:go_default_library",
+        "//vendor/github.com/go-vgo/gt/conf:go_default_library",
+        "//vendor/github.com/go-vgo/gt/info:go_default_library",
+        "//vendor/github.com/shirou/gopsutil/mem:go_default_library",
+        "//vendor/github.com/syndtr/goleveldb/leveldb:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//vendor/github.com/go-ego/riot/core:all-srcs",
+        "//vendor/github.com/go-ego/riot/store:all-srcs",
+        "//vendor/github.com/go-ego/riot/types:all-srcs",
+        "//vendor/github.com/go-ego/riot/utils:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/go-ego/riot/CONTRIBUTING.md

@@ -0,0 +1,73 @@
+# Contribution Guidelines
+
+## Introduction
+
+This document explains how to contribute changes to the Ego project. It assumes you have followed the README.md and [API Document](https://github.com/go-ego/riot/tree/master/docs). <!--Sensitive security-related issues should be reported to [security@Ego.io](mailto:security@Ego.io.)-->
+
+## Bug reports
+
+Please search the issues on the issue tracker with a variety of keywords to ensure your bug is not already reported.
+
+If unique, [open an issue](https://github.com/go-ego/riot/issues/new) and answer the questions so we can understand and reproduce the problematic behavior.
+
+The burden is on you to convince us that it is actually a bug in Ego. This is easiest to do when you write clear, concise instructions so we can reproduce the behavior (even if it seems obvious). The more detailed and specific you are, the faster we will be able to help you. Check out [How to Report Bugs Effectively](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html).
+
+Please be kind, remember that Ego comes at no cost to you, and you're getting free help.
+
+## Discuss your design
+
+The project welcomes submissions but please let everyone know what you're working on if you want to change or add something to the Ego repositories.
+
+Before starting to write something new for the Ego project, please [file an issue](https://github.com/go-ego/riot/issues/new). Significant changes must go through the [change proposal process](https://github.com/go-ego/proposals) before they can be accepted.
+
+This process gives everyone a chance to validate the design, helps prevent duplication of effort, and ensures that the idea fits inside the goals for the project and tools. It also checks that the design is sound before code is written; the code review tool is not the place for high-level discussions.
+
+## Testing redux
+
+Before sending code out for review, run all the tests for the whole tree to make sure the changes don't break other usage and keep the compatibility on upgrade. You must be test on Mac, Windows, Linux and other. You should install the CLI for Circle CI, as we are using the server for continous testing.
+
+## Code review
+
+In addition to the owner, Changes to Ego must be reviewed before they are accepted, no matter who makes the change even if it is a maintainer. We use GitHub's pull request workflow to do that and we also use [LGTM](http://lgtm.co) to ensure every PR is reviewed by vz or least 2 maintainers.
+
+
+## Sign your work
+
+The sign-off is a simple line at the end of the explanation for the patch. Your signature certifies that you wrote the patch or otherwise have the right to pass it on as an open-source patch. 
+
+## Maintainers
+
+To make sure every PR is checked, we got team maintainers. A maintainer should be a contributor of Ego and contributed at least 4 accepted PRs. 
+
+## Owners
+
+Since Ego is a pure community organization without any company support, Copyright 2016 The go-ego Project Developers.
+
+
+## Versions
+
+Ego has the `master` branch as a tip branch and has version branches such as `v0.30.0`. `v0.40.0` is a release branch and we will tag `v0.40.0` for binary download. If `v0.40.0` has bugs, we will accept pull requests on the `v0.40.0` branch and publish a `v0.40.1` tag, after bringing the bug fix also to the master branch.
+
+Since the `master` branch is a tip version, if you wish to use Ego in production, please download the latest release tag version. All the branches will be protected via GitHub, all the PRs to every branch must be reviewed by two maintainers and must pass the automatic tests.
+
+## Copyright
+
+Code that you contribute should use the standard copyright header:
+
+```
+// Copyright 2016 The go-ego Project Developers. 
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+```
+
+Files in the repository contain copyright from the year they are added to the year they are last changed. If the copyright author is changed, just paste the header below the old one.

vendor/github.com/go-ego/riot/Gopkg.lock

@@ -0,0 +1,323 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  branch = "master"
+  name = "github.com/AndreasBriese/bbloom"
+  packages = ["."]
+  revision = "28f7e881ca57bc00e028f9ede9f0d9104cfeef5e"
+
+[[projects]]
+  name = "github.com/BurntSushi/toml"
+  packages = ["."]
+  revision = "b26d9c308763d68093482582cea63d69be07a0f0"
+  version = "v0.3.0"
+
+[[projects]]
+  name = "github.com/StackExchange/wmi"
+  packages = ["."]
+  revision = "5d049714c4a64225c3c79a7cf7d02f7fb5b96338"
+  version = "1.0.0"
+
+[[projects]]
+  name = "github.com/coreos/bbolt"
+  packages = ["."]
+  revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9"
+  version = "v1.3.0"
+
+[[projects]]
+  name = "github.com/coreos/etcd"
+  packages = [
+    "auth/authpb",
+    "clientv3",
+    "etcdserver/api/v3rpc/rpctypes",
+    "etcdserver/etcdserverpb",
+    "mvcc/mvccpb",
+    "pkg/types"
+  ]
+  revision = "fca8add78a9d926166eb739b8e4a124434025ba3"
+  version = "v3.3.9"
+
+[[projects]]
+  name = "github.com/dgraph-io/badger"
+  packages = [
+    ".",
+    "options",
+    "protos",
+    "skl",
+    "table",
+    "y"
+  ]
+  revision = "391b6d3b93e6014fe8c2971fcc0c1266e47dbbd9"
+  version = "v1.5.3"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/dgryski/go-farm"
+  packages = ["."]
+  revision = "2de33835d10275975374b37b2dcfd22c9020a1f5"
+
+[[projects]]
+  name = "github.com/fsnotify/fsnotify"
+  packages = ["."]
+  revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
+  version = "v1.4.7"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/go-ego/cedar"
+  packages = ["."]
+  revision = "39a3301a49c052d18c55ade2bd080cda1ccc0446"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/go-ego/gpy"
+  packages = ["."]
+  revision = "5e7198f4b498342cd0e29af3b199470cbbcb7c08"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/go-ego/gse"
+  packages = ["."]
+  revision = "78fb4a3717d7d60731b5b95e651e86325419c064"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/go-ego/murmur"
+  packages = ["."]
+  revision = "0e80721bb094033c73e7b6159d3c1cbd116e9b38"
+
+[[projects]]
+  name = "github.com/go-ole/go-ole"
+  packages = [
+    ".",
+    "oleutil"
+  ]
+  revision = "a41e3c4b706f6ae8dfbff342b06e40fa4d2d0506"
+  version = "v1.2.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/go-vgo/grpclb"
+  packages = ["."]
+  revision = "434b4da1cea2111c03df2a91cc96cbff17110e43"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/go-vgo/gt"
+  packages = [
+    "conf",
+    "info",
+    "zlog"
+  ]
+  revision = "765190081bade7de90c53a7117f1c6a3dee5b0de"
+
+[[projects]]
+  name = "github.com/gogo/protobuf"
+  packages = [
+    "gogoproto",
+    "proto",
+    "protoc-gen-gogo/descriptor"
+  ]
+  revision = "636bf0302bc95575d69441b25a2603156ffdddf1"
+  version = "v1.1.1"
+
+[[projects]]
+  name = "github.com/golang/protobuf"
+  packages = [
+    "proto",
+    "protoc-gen-go/descriptor",
+    "ptypes",
+    "ptypes/any",
+    "ptypes/duration",
+    "ptypes/timestamp"
+  ]
+  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
+  version = "v1.1.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/snappy"
+  packages = ["."]
+  revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
+
+[[projects]]
+  name = "github.com/pelletier/go-toml"
+  packages = ["."]
+  revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
+  version = "v1.2.0"
+
+[[projects]]
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  version = "v0.8.0"
+
+[[projects]]
+  name = "github.com/shirou/gopsutil"
+  packages = [
+    "cpu",
+    "disk",
+    "host",
+    "internal/common",
+    "mem",
+    "net",
+    "process"
+  ]
+  revision = "8048a2e9c5773235122027dd585cf821b2af1249"
+  version = "v2.18.07"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/shirou/w32"
+  packages = ["."]
+  revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/syndtr/goleveldb"
+  packages = [
+    "leveldb",
+    "leveldb/cache",
+    "leveldb/comparer",
+    "leveldb/errors",
+    "leveldb/filter",
+    "leveldb/iterator",
+    "leveldb/journal",
+    "leveldb/memdb",
+    "leveldb/opt",
+    "leveldb/storage",
+    "leveldb/table",
+    "leveldb/util"
+  ]
+  revision = "c4c61651e9e37fa117f53c5a906d3b63090d8445"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/vcaesar/tt"
+  packages = ["."]
+  revision = "bafafb004f106551bfbcb6cac4ec3f9932365a57"
+
+[[projects]]
+  name = "go.uber.org/atomic"
+  packages = ["."]
+  revision = "1ea20fb1cbb1cc08cbd0d913a96dead89aa18289"
+  version = "v1.3.2"
+
+[[projects]]
+  name = "go.uber.org/multierr"
+  packages = ["."]
+  revision = "3c4937480c32f4c13a875a1829af76c98ca3d40a"
+  version = "v1.1.0"
+
+[[projects]]
+  name = "go.uber.org/zap"
+  packages = [
+    ".",
+    "buffer",
+    "internal/bufferpool",
+    "internal/color",
+    "internal/exit",
+    "zapcore"
+  ]
+  revision = "4d45f9617f7d90f7a663ff21c7a4321dbe78098b"
+  version = "v1.9.0"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/net"
+  packages = [
+    "context",
+    "http/httpguts",
+    "http2",
+    "http2/hpack",
+    "idna",
+    "internal/timeseries",
+    "trace"
+  ]
+  revision = "32f9bdbd7df18e8641d215e7ea68be88b971feb0"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/sys"
+  packages = [
+    "unix",
+    "windows"
+  ]
+  revision = "bd9dbc187b6e1dacfdd2722a87e83093c2d7bd6e"
+
+[[projects]]
+  name = "golang.org/x/text"
+  packages = [
+    "collate",
+    "collate/build",
+    "internal/colltab",
+    "internal/gen",
+    "internal/tag",
+    "internal/triegen",
+    "internal/ucd",
+    "language",
+    "secure/bidirule",
+    "transform",
+    "unicode/bidi",
+    "unicode/cldr",
+    "unicode/norm",
+    "unicode/rangetable"
+  ]
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
+
+[[projects]]
+  branch = "master"
+  name = "google.golang.org/genproto"
+  packages = ["googleapis/rpc/status"]
+  revision = "daca94659cb50e9f37c1b834680f2e46358f10b0"
+
+[[projects]]
+  name = "google.golang.org/grpc"
+  packages = [
+    ".",
+    "balancer",
+    "balancer/base",
+    "balancer/roundrobin",
+    "codes",
+    "connectivity",
+    "credentials",
+    "encoding",
+    "encoding/proto",
+    "grpclog",
+    "health/grpc_health_v1",
+    "internal",
+    "internal/backoff",
+    "internal/channelz",
+    "internal/envconfig",
+    "internal/grpcrand",
+    "internal/transport",
+    "keepalive",
+    "metadata",
+    "naming",
+    "peer",
+    "reflection",
+    "reflection/grpc_reflection_v1alpha",
+    "resolver",
+    "resolver/dns",
+    "resolver/passthrough",
+    "stats",
+    "status",
+    "tap"
+  ]
+  revision = "32fb0ac620c32ba40a4626ddf94d90d12cce3455"
+  version = "v1.14.0"
+
+[[projects]]
+  name = "gopkg.in/natefinch/lumberjack.v2"
+  packages = ["."]
+  revision = "a96e63847dc3c67d17befa69c303767e2f84e54f"
+  version = "v2.1"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "bbf0b187aabef1ad7a86a7870838007190e413053fa4f3063da4f2faa34b671e"
+  solver-name = "gps-cdcl"
+  solver-version = 1

vendor/github.com/go-ego/riot/Gopkg.toml

@@ -0,0 +1,82 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[[constraint]]
+  name = "github.com/coreos/bbolt"
+  version = "1.3.0"
+
+[[constraint]]
+  name = "github.com/dgraph-io/badger"
+  version = "1.5.1"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/go-ego/gpy"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/go-ego/gse"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/go-ego/murmur"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/go-vgo/grpclb"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/go-vgo/gt"
+
+[[constraint]]
+  name = "github.com/gogo/protobuf"
+  version = "1.0.0"
+
+[[constraint]]
+  name = "github.com/shirou/gopsutil"
+  version = "2.18.05"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/syndtr/goleveldb"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/vcaesar/tt"
+
+[[constraint]]
+  branch = "master"
+  name = "golang.org/x/net"
+
+[[constraint]]
+  name = "google.golang.org/grpc"
+  version = "1.12.2"
+
+[prune]
+  go-tests = true
+  unused-packages = true

vendor/github.com/go-ego/riot/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/go-ego/riot/README.md

@@ -0,0 +1,178 @@
+# Riot search
+
+<img align="right" src="logo/512px.svg" width="15%"/>
+
+<!--<img align="right" src="https://raw.githubusercontent.com/go-ego/ego/master/logo.jpg">-->
+<!--<a href="https://circleci.com/gh/go-ego/ego/tree/dev"><img src="https://img.shields.io/circleci/project/go-ego/ego/dev.svg" alt="Build Status"></a>-->
+[![CircleCI Status](https://circleci.com/gh/go-ego/riot.svg?style=shield)](https://circleci.com/gh/go-ego/riot)
+![Appveyor](https://ci.appveyor.com/api/projects/status/github/go-ego/riot?branch=master&svg=true)
+[![codecov](https://codecov.io/gh/go-ego/riot/branch/master/graph/badge.svg)](https://codecov.io/gh/go-ego/riot)
+[![Build Status](https://travis-ci.org/go-ego/riot.svg)](https://travis-ci.org/go-ego/riot)
+[![Go Report Card](https://goreportcard.com/badge/github.com/go-ego/riot)](https://goreportcard.com/report/github.com/go-ego/riot)
+[![GoDoc](https://godoc.org/github.com/go-ego/riot?status.svg)](https://godoc.org/github.com/go-ego/riot)
+[![Release](https://github-release-version.herokuapp.com/github/go-ego/riot/release.svg?style=flat)](https://github.com/go-ego/riot/releases/latest)
+[![Join the chat at https://gitter.im/go-ego/ego](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/go-ego/ego?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+<!--<a href="https://github.com/go-ego/ego/releases"><img src="https://img.shields.io/badge/%20version%20-%206.0.0%20-blue.svg?style=flat-square" alt="Releases"></a>-->
+
+<!-- ![ego Logo](logo/512px.svg) -->
+Go Open Source, Distributed, Simple and efficient full text search engine.
+
+[简体中文](https://github.com/go-ego/riot/blob/master/README_zh.md)
+
+# Features
+
+* [Efficient indexing and search](/docs/en/benchmarking.md) (1M blog 500M data 28 seconds index finished, 1.65 ms search response time, 19K search QPS）
+* Support for [logical search](https://github.com/go-ego/riot/blob/master/docs/en/logic.md)
+* Support Chinese word segmentation (use [gse word segmentation package](https://github.com/go-ego/gse) concurrent word, speed 27MB / s）
+* Support the calculation of the keyword in the text [close to the distance](/docs/en/token_proximity.md)（token proximity）
+* Support calculation [BM25 correlation](/docs/en/bm25.md)
+* Support [custom scoring field and scoring rules](/docs/en/custom_scoring_criteria.md)
+* Support [add online, delete index](/docs/en/realtime_indexing.md)
+* Support heartbeat
+* Support multiple [persistent storage](/docs/en/persistent_storage.md)
+* Support [distributed index and search](https://github.com/go-ego/riot/tree/master/data)
+* Can be achieved [distributed index and search](/docs/en/distributed_indexing_and_search.md)
+
+* [Look at Word segmentation rules](https://github.com/go-ego/riot/blob/master/docs/en/segmenter.md)
+
+
+Riot v0.10.0 was released in Nov 2017, check the [Changelog](https://github.com/go-ego/riot/blob/master/docs/CHANGELOG.md) for the full details.
+
+## Requirements
+Go version >= 1.8
+
+### Vendored Dependencies
+
+Riot uses [dep](https://github.com/golang/dep) to vendor dependencies, but we don't commit the vendored packages themselves to the Riot git repository. Therefore, a simple go get is not supported because the command is not vendor aware. 
+
+Please manage it with dep, run `dep ensure`, to clone dependencies.
+
+## Installation/Update
+
+```
+go get -u github.com/go-ego/riot
+```
+
+## [Build-tools](https://github.com/go-ego/re)
+```
+go get -u github.com/go-ego/re 
+```
+### re riot
+To create a new riot application
+
+```
+$ re riot my-riotapp
+```
+
+### re run
+
+To run the application we just created, you can navigate to the application folder and execute:
+```
+$ cd my-riotapp && re run
+```
+
+## Usage:
+
+#### [Look at an example](/examples/simple/main.go)
+
+```go
+package main
+
+import (
+	"log"
+
+	"github.com/go-ego/riot"
+	"github.com/go-ego/riot/types"
+)
+
+var (
+	// searcher is coroutine safe
+	searcher = riot.Engine{}
+)
+
+func main() {
+	// Init
+	searcher.Init(types.EngineOpts{
+		// Using:             4,
+		NotUseGse: true,
+		})
+	defer searcher.Close()
+
+	text := "Google Is Experimenting With Virtual Reality Advertising"
+	text1 := `Google accidentally pushed Bluetooth update for Home
+	speaker early`
+	text2 := `Google is testing another Search results layout with 
+	rounded cards, new colors, and the 4 mysterious colored dots again`
+	
+	// Add the document to the index, docId starts at 1
+	searcher.Index(1, types.DocData{Content: text})
+	searcher.Index(2, types.DocData{Content: text1}, false)
+	searcher.IndexDoc(3, types.DocData{Content: text2}, true)
+
+	// Wait for the index to refresh
+	searcher.Flush()
+	// engine.FlushIndex()
+
+	// The search output format is found in the types.SearchResp structure
+	log.Print(searcher.Search(types.SearchReq{Text:"google testing"}))
+}
+```
+
+It is very simple!
+
+### Use default engine:
+
+```Go
+package main
+
+import (
+	"log"
+
+	"github.com/go-ego/riot"
+	"github.com/go-ego/riot/types"
+)
+
+var (
+	searcher = riot.New("zh")
+)
+
+func main() {
+	data := types.DocData{Content: `I wonder how, I wonder why
+		, I wonder where they are`}
+	data1 := types.DocData{Content: "所以, 你好, 再见"}
+	data2 := types.DocData{Content: "没有理由"}
+	searcher.Index(1, data)
+	searcher.Index(2, data1)
+	searcher.Index(3, data2)
+	searcher.Flush()
+
+	req := types.SearchReq{Text: "你好"}
+	search := searcher.Search(req)
+	log.Println("search...", search)
+}
+```
+
+#### [Look at more Examples](https://github.com/go-ego/riot/tree/master/examples)
+
+#### [Look at Store example](https://github.com/go-ego/riot/blob/master/examples/store/main.go)
+#### [Look at Logic search example](https://github.com/go-ego/riot/blob/master/examples/logic/main.go)
+
+#### [Look at Pinyin search example](https://github.com/go-ego/riot/blob/master/examples/pinyin/main.go)
+
+#### [Look at different dict and language search example](https://github.com/go-ego/riot/blob/master/examples/dict/main.go)
+
+#### [Look at benchmark example](https://github.com/go-ego/riot/blob/master/examples/benchmark/benchmark.go)
+
+#### [Riot search engine templates, client and dictionaries](https://github.com/go-ego/riot/tree/master/data)
+
+## Donate
+
+Supporting riot, [buy me a coffee](https://github.com/go-vgo/buy-me-a-coffee).
+
+#### Paypal
+
+Donate money by [paypal](https://www.paypal.me/veni0/25) to my account [vzvway@gmail.com](vzvway@gmail.com)
+
+## License
+
+Riot is primarily distributed under the terms of the Apache License (Version 2.0), base on [wukong](https://github.com/huichen/wukong).

vendor/github.com/go-ego/riot/README_zh.md

@@ -0,0 +1,194 @@
+# [Riot 搜索引擎](https://github.com/go-ego/riot)
+
+<!--<img align="right" src="https://raw.githubusercontent.com/go-ego/ego/master/logo.jpg">-->
+<!--<a href="https://circleci.com/gh/go-ego/ego/tree/dev"><img src="https://img.shields.io/circleci/project/go-ego/ego/dev.svg" alt="Build Status"></a>-->
+[![CircleCI Status](https://circleci.com/gh/go-ego/riot.svg?style=shield)](https://circleci.com/gh/go-ego/riot)
+![Appveyor](https://ci.appveyor.com/api/projects/status/github/go-ego/riot?branch=master&svg=true)
+[![codecov](https://codecov.io/gh/go-ego/riot/branch/master/graph/badge.svg)](https://codecov.io/gh/go-ego/riot)
+[![Build Status](https://travis-ci.org/go-ego/riot.svg)](https://travis-ci.org/go-ego/riot)
+[![Go Report Card](https://goreportcard.com/badge/github.com/go-ego/riot)](https://goreportcard.com/report/github.com/go-ego/riot)
+[![GoDoc](https://godoc.org/github.com/go-ego/riot?status.svg)](https://godoc.org/github.com/go-ego/riot)
+[![Release](https://github-release-version.herokuapp.com/github/go-ego/riot/release.svg?style=flat)](https://github.com/go-ego/riot/releases/latest)
+[![Join the chat at https://gitter.im/go-ego/ego](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/go-ego/ego?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+<!--<a href="https://github.com/go-ego/ego/releases"><img src="https://img.shields.io/badge/%20version%20-%206.0.0%20-blue.svg?style=flat-square" alt="Releases"></a>-->
+
+
+Go Open Source, Distributed, Simple and efficient full text search engine.
+
+# Features
+
+* [高效索引和搜索](/docs/zh/benchmarking.md)（1M 条微博 500M 数据28秒索引完，1.65毫秒搜索响应时间，19K 搜索 QPS）
+* 支持中文分词（使用 [gse 分词包](https://github.com/go-ego/gse)并发分词，速度 27MB/秒）
+* 支持[逻辑搜索](https://github.com/go-ego/riot/blob/master/docs/zh/logic.md)
+* 支持中文转拼音搜索(使用 [gpy](https://github.com/go-ego/gpy) 中文转拼音)
+* 支持计算关键词在文本中的[紧邻距离](/docs/zh/token_proximity.md)（token proximity）
+* 支持计算[BM25相关度](/docs/zh/bm25.md)
+* 支持[自定义评分字段和评分规则](/docs/zh/custom_scoring_criteria.md)
+* 支持[在线添加、删除索引](/docs/zh/realtime_indexing.md)
+* 支持多种[持久存储](/docs/zh/persistent_storage.md)
+* 支持 heartbeat
+* 支持[分布式索引和搜索](https://github.com/go-ego/riot/tree/master/data)
+* 可实现[分布式索引和搜索](/docs/zh/distributed_indexing_and_search.md)
+* 采用对商业应用友好的[Apache License v2](/LICENSE)发布
+
+* [查看分词规则](https://github.com/go-ego/riot/blob/master/docs/zh/segmenter.md)
+
+Riot v0.10.0 was released in Nov 2017, check the [Changelog](https://github.com/go-ego/riot/blob/master/docs/CHANGELOG.md) for the full details.
+
+QQ 群: 120563750
+
+## 安装/更新
+
+```
+go get -u github.com/go-ego/riot
+```
+
+## Requirements
+
+需要 Go 版本至少 1.8
+
+### Vendored Dependencies
+
+Riot 使用 [dep](https://github.com/golang/dep) 管理 vendor 依赖, but we don't commit the vendored packages themselves to the Riot git repository. Therefore, a simple go get is not supported because the command is not vendor aware. 
+
+请用 dep 管理它, 运行 `dep ensure` 克隆依赖.
+
+## [Build-tools](https://github.com/go-ego/re)
+```
+go get -u github.com/go-ego/re 
+```
+### re riot
+创建 riot 项目
+
+```
+$ re riot my-riotapp
+```
+
+### re run
+
+运行我们创建的 riot 项目, 你可以导航到应用程序文件夹并执行:
+```
+$ cd my-riotapp && re run
+```
+
+## 使用
+
+先看一个例子（来自 [simplest_example.go](/examples/simple/zh/main.go)）
+```go
+package main
+
+import (
+	"log"
+
+	"github.com/go-ego/riot"
+	"github.com/go-ego/riot/types"
+)
+
+var (
+	// searcher 是协程安全的
+	searcher = riot.Engine{}
+)
+
+func main() {
+	// 初始化
+	searcher.Init(types.EngineOpts{
+		Using:             3,
+		GseDict: "zh",
+		// GseDict: "your gopath"+"/src/github.com/go-ego/riot/data/dict/dictionary.txt",
+	})
+	defer searcher.Close()
+
+	text := "此次百度收购将成中国互联网最大并购"
+	text1 := "百度宣布拟全资收购91无线业务"
+	text2 := "百度是中国最大的搜索引擎"
+	
+	// 将文档加入索引，docId 从1开始
+	searcher.Index(1, types.DocData{Content: text})
+	searcher.Index(2, types.DocData{Content: text1}, false)
+	searcher.Index(3, types.DocData{Content: text2}, true)
+
+	// 等待索引刷新完毕
+	searcher.Flush()
+	// engine.FlushIndex()
+
+	// 搜索输出格式见 types.SearchResp 结构体
+	log.Print(searcher.Search(types.SearchReq{Text:"百度中国"}))
+}
+```
+
+是不是很简单！
+
+然后看看一个[入门教程](/docs/zh/codelab.md)，教你用不到200行 Go 代码实现一个微博搜索网站。
+
+### 使用默认引擎:
+
+```Go
+package main
+
+import (
+	"log"
+
+	"github.com/go-ego/riot"
+	"github.com/go-ego/riot/types"
+)
+
+var (
+	searcher = riot.New("zh")
+)
+
+func main() {
+	data := types.DocData{Content: `I wonder how, I wonder why
+		, I wonder where they are`}
+	data1 := types.DocData{Content: "所以, 你好, 再见"}
+	data2 := types.DocData{Content: "没有理由"}
+	searcher.Index(1, data)
+	searcher.Index(2, data1)
+	searcher.IndexDoc(3, data2)
+	searcher.Flush()
+
+	req := types.SearchReq{Text: "你好"}
+	search := searcher.Search(req)
+	log.Println("search...", search)
+}
+```
+
+#### [查看更多例子](https://github.com/go-ego/riot/tree/master/examples)
+
+#### [持久化的例子](https://github.com/go-ego/riot/blob/master/examples/store/main.go)
+#### [逻辑搜索的例子](https://github.com/go-ego/riot/blob/master/examples/logic/main.go)
+
+#### [拼音搜索的例子](https://github.com/go-ego/riot/blob/master/examples/pinyin/main.go)
+
+#### [不同字典和语言例子](https://github.com/go-ego/riot/blob/master/examples/dict/main.go)
+
+#### [benchmark](https://github.com/go-ego/riot/blob/master/examples/benchmark/benchmark.go)
+
+#### [Riot 搜索模板, 客户端和字典](https://github.com/go-ego/riot/tree/master/data)
+
+## 主要改进:
+
+- 增加逻辑搜索 
+- 增加拼音搜索 
+- 增加分布式 
+- 分词等改进 
+- 增加更多 api
+- 支持 heartbeat
+- 修复 bug
+- 删除依赖 cgo 的存储引擎, 增加 badger和 leveldb 持久化引擎
+
+## Donate
+
+支持 riot, [buy me a coffee](https://github.com/go-vgo/buy-me-a-coffee).
+
+#### Paypal
+
+Donate money by [paypal](https://www.paypal.me/veni0/25) to my account [vzvway@gmail.com](vzvway@gmail.com)
+
+## 其它
+
+* [为什么要有 riot 引擎](/docs/zh/why_riot.md)
+* [联系方式](/docs/zh/feedback.md)
+
+## License
+
+Riot is primarily distributed under the terms of the Apache License (Version 2.0), base on [wukong](https://github.com/huichen/wukong).

vendor/github.com/go-ego/riot/appveyor.yml

@@ -0,0 +1,42 @@
+version: "{build}"
+image: 'Visual Studio 2017'
+# platform: x64
+
+clone_folder: c:\gopath\src\github.com\go-ego\riot
+# max_jobs: 1
+
+environment:
+  global:
+    GOPATH: C:\gopath
+    # CC: gcc.exe
+  matrix:
+    - GOARCH: amd64
+    # - GOARCH: 386
+  GOVERSION: 1.10.3
+  # GOPATH: c:\gopath
+
+install:
+  - set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
+  - git submodule update --init
+  - rmdir C:\go /s /q
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go%GOVERSION%.windows-%GOARCH%.zip
+  - 7z x go%GOVERSION%.windows-%GOARCH%.zip -y -oC:\ > NUL
+  - go version
+  - go env
+  # - gcc --version  
+
+# To run your custom scripts instead of automatic MSBuild
+build_script:
+  # We need to disable firewall - https://github.com/appveyor/ci/issues/1579#issuecomment-309830648
+  - ps: Disable-NetFirewallRule -DisplayName 'File and Printer Sharing (SMB-Out)'
+  - cd c:\gopath\src\github.com\go-ego\riot
+  - git branch
+  - go get -t ./...
+
+# To run your custom scripts instead of automatic tests
+test_script:
+  # Unit tests
+  - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
+  # - go test -u github.com/go-ego/riot/...
+  - go test -v github.com/go-ego/riot/...
+  - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed

vendor/github.com/go-ego/riot/circle.yml

@@ -0,0 +1,34 @@
+# circle.yml #
+# machine:
+#   go:
+#     version: 1.9.1
+
+version: 2
+
+jobs:
+  build:
+    docker:
+      - image: govgo/go:1.10.3
+    working_directory: /gopath/src/github.com/go-ego/riot
+    steps:
+      - checkout
+      # specify any bash command here prefixed with `run: `
+      # - run: go get -u github.com/go-ego/gse  
+      # - run: go get -u github.com/go-ego/gpy 
+      # - run: go get -u github.com/go-ego/murmur 
+      # - run: go get -u golang.org/x/sys/unix
+      # - run: go get -u github.com/shirou/gopsutil
+      - run: go get -v -t -d ./...
+      - run: go test -v ./... 
+       # codecov.io
+      - run: go test -v -covermode=count -coverprofile=coverage.out
+      - run: bash <(curl -s https://codecov.io/bash)   
+
+# script:
+#   - ./go.test.sh
+
+# test:
+#   post:
+#     - go test -v -covermode=count -coverprofile=coverage.out
+#     - bash <(curl -s https://codecov.io/bash) 
+   

vendor/github.com/go-ego/riot/core/BUILD.bazel

@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "data.go",
+        "indexer.go",
+        "ranker.go",
+        "test_utils.go",
+        "uint64.go",
+    ],
+    importmap = "go-common/vendor/github.com/go-ego/riot/core",
+    importpath = "github.com/go-ego/riot/core",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/go-ego/riot/types:go_default_library",
+        "//vendor/github.com/go-ego/riot/utils:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/go-ego/riot/core/data.go

@@ -0,0 +1,94 @@
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package core
+
+import (
+	"sync"
+
+	"github.com/go-ego/riot/types"
+)
+
+var (
+	// DocInfoGroup 文档信息 [shard][id]info
+	DocInfoGroup         = make(map[int]*types.DocInfosShard)
+	docInfosGroupRWMutex sync.RWMutex
+)
+
+// AddDocInfosShard add document infos shard
+func AddDocInfosShard(shard int) {
+	docInfosGroupRWMutex.Lock()
+	defer docInfosGroupRWMutex.Unlock()
+	if _, found := DocInfoGroup[shard]; !found {
+		DocInfoGroup[shard] = &types.DocInfosShard{
+			DocInfos: make(map[uint64]*types.DocInfo),
+		}
+	}
+}
+
+// AddDocInfo add documents info
+func AddDocInfo(shard int, docId uint64, docinfo *types.DocInfo) {
+	docInfosGroupRWMutex.Lock()
+	defer docInfosGroupRWMutex.Unlock()
+	if _, ok := DocInfoGroup[shard]; !ok {
+		DocInfoGroup[shard] = &types.DocInfosShard{
+			DocInfos: make(map[uint64]*types.DocInfo),
+		}
+	}
+	DocInfoGroup[shard].DocInfos[docId] = docinfo
+	DocInfoGroup[shard].NumDocs++
+}
+
+// IsDocExist doc is exist
+func IsDocExist(docId uint64) bool {
+	docInfosGroupRWMutex.RLock()
+	defer docInfosGroupRWMutex.RUnlock()
+	for _, docInfosShard := range DocInfoGroup {
+		_, found := docInfosShard.DocInfos[docId]
+		if found {
+			return true
+		}
+	}
+	return false
+}
+
+var (
+	// InvertedIndexGroup 反向索引表([shard][关键词]反向索引表)
+	InvertedIndexGroup        = make(map[int]*types.InvertedIndexShard)
+	invertedIndexGroupRWMutex sync.RWMutex
+)
+
+// AddInvertedIndexShard add inverted index shard
+func AddInvertedIndexShard(shard int) {
+	invertedIndexGroupRWMutex.Lock()
+	defer invertedIndexGroupRWMutex.Unlock()
+	if _, found := InvertedIndexGroup[shard]; !found {
+		InvertedIndexGroup[shard] = &types.InvertedIndexShard{
+			InvertedIndex: make(map[string]*types.KeywordIndices),
+		}
+	}
+}
+
+// AddKeywordIndices  add keyword indices
+func AddKeywordIndices(shard int, keyword string, keywordIndices *types.KeywordIndices) {
+	invertedIndexGroupRWMutex.Lock()
+	defer invertedIndexGroupRWMutex.Unlock()
+	if _, ok := InvertedIndexGroup[shard]; !ok {
+		InvertedIndexGroup[shard] = &types.InvertedIndexShard{
+			InvertedIndex: make(map[string]*types.KeywordIndices),
+		}
+	}
+	InvertedIndexGroup[shard].InvertedIndex[keyword] = keywordIndices
+	InvertedIndexGroup[shard].TotalTokenLen++
+}

vendor/github.com/go-ego/riot/core/indexer.go

@@ -0,0 +1,881 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+/*
+
+Package core is riot core
+*/
+package core
+
+import (
+	"log"
+	"math"
+	"sort"
+	"sync"
+
+	"github.com/go-ego/riot/types"
+	"github.com/go-ego/riot/utils"
+)
+
+// Indexer 索引器
+type Indexer struct {
+	// 从搜索键到文档列表的反向索引
+	// 加了读写锁以保证读写安全
+	tableLock struct {
+		sync.RWMutex
+		table     map[string]*KeywordIndices
+		docsState map[uint64]int // nil: 表示无状态记录，0: 存在于索引中，1: 等待删除，2: 等待加入
+	}
+
+	addCacheLock struct {
+		sync.RWMutex
+		addCachePointer int
+		addCache        types.DocsIndex
+	}
+
+	removeCacheLock struct {
+		sync.RWMutex
+		removeCachePointer int
+		removeCache        types.DocsId
+	}
+
+	initOptions types.IndexerOpts
+	initialized bool
+
+	// 这实际上是总文档数的一个近似
+	numDocs uint64
+
+	// 所有被索引文本的总关键词数
+	totalTokenLen float32
+
+	// 每个文档的关键词长度
+	docTokenLens map[uint64]float32
+}
+
+// KeywordIndices 反向索引表的一行，收集了一个搜索键出现的所有文档，按照DocId从小到大排序。
+type KeywordIndices struct {
+	// 下面的切片是否为空，取决于初始化时IndexType的值
+	docIds      []uint64  // 全部类型都有
+	frequencies []float32 // IndexType == FrequenciesIndex
+	locations   [][]int   // IndexType == LocsIndex
+}
+
+// Init 初始化索引器
+func (indexer *Indexer) Init(options types.IndexerOpts) {
+	if indexer.initialized == true {
+		log.Fatal("The Indexer can not be initialized twice.")
+	}
+	options.Init()
+	indexer.initOptions = options
+	indexer.initialized = true
+
+	indexer.tableLock.table = make(map[string]*KeywordIndices)
+	indexer.tableLock.docsState = make(map[uint64]int)
+	indexer.addCacheLock.addCache = make(
+		[]*types.DocIndex, indexer.initOptions.DocCacheSize)
+
+	indexer.removeCacheLock.removeCache = make(
+		[]uint64, indexer.initOptions.DocCacheSize*2)
+	indexer.docTokenLens = make(map[uint64]float32)
+}
+
+// getDocId 从 KeywordIndices 中得到第i个文档的 DocId
+func (indexer *Indexer) getDocId(ti *KeywordIndices, i int) uint64 {
+	return ti.docIds[i]
+}
+
+// HasDoc doc is exist return true
+func (indexer *Indexer) HasDoc(docId uint64) bool {
+	docState, ok := indexer.tableLock.docsState[docId]
+	if ok && docState == 0 {
+		return true
+	}
+
+	return false
+}
+
+// getIndexLen 得到 KeywordIndices 中文档总数
+func (indexer *Indexer) getIndexLen(ti *KeywordIndices) int {
+	return len(ti.docIds)
+}
+
+// AddDocToCache 向 ADDCACHE 中加入一个文档
+func (indexer *Indexer) AddDocToCache(doc *types.DocIndex, forceUpdate bool) {
+	if indexer.initialized == false {
+		log.Fatal("The Indexer has not been initialized.")
+	}
+
+	indexer.addCacheLock.Lock()
+	if doc != nil {
+		indexer.addCacheLock.addCache[indexer.addCacheLock.addCachePointer] = doc
+		indexer.addCacheLock.addCachePointer++
+	}
+
+	if indexer.addCacheLock.addCachePointer >= indexer.initOptions.DocCacheSize ||
+		forceUpdate {
+		indexer.tableLock.Lock()
+
+		position := 0
+		for i := 0; i < indexer.addCacheLock.addCachePointer; i++ {
+			docIndex := indexer.addCacheLock.addCache[i]
+
+			docState, ok := indexer.tableLock.docsState[docIndex.DocId]
+			if ok && docState <= 1 {
+				// ok && docState == 0 表示存在于索引中，需先删除再添加
+				// ok && docState == 1 表示不一定存在于索引中，等待删除，需先删除再添加
+				if position != i {
+					indexer.addCacheLock.addCache[position], indexer.addCacheLock.addCache[i] =
+						indexer.addCacheLock.addCache[i], indexer.addCacheLock.addCache[position]
+				}
+				if docState == 0 {
+					indexer.removeCacheLock.Lock()
+					indexer.removeCacheLock.removeCache[indexer.removeCacheLock.removeCachePointer] =
+						docIndex.DocId
+					indexer.removeCacheLock.removeCachePointer++
+					indexer.removeCacheLock.Unlock()
+
+					indexer.tableLock.docsState[docIndex.DocId] = 1
+					indexer.numDocs--
+				}
+				position++
+			} else if !ok {
+				indexer.tableLock.docsState[docIndex.DocId] = 2
+			}
+		}
+
+		indexer.tableLock.Unlock()
+		if indexer.RemoveDocToCache(0, forceUpdate) {
+			// 只有当存在于索引表中的文档已被删除，其才可以重新加入到索引表中
+			position = 0
+		}
+
+		addCachedDocs := indexer.addCacheLock.addCache[position:indexer.addCacheLock.addCachePointer]
+		indexer.addCacheLock.addCachePointer = position
+
+		indexer.addCacheLock.Unlock()
+		sort.Sort(addCachedDocs)
+		indexer.AddDocs(&addCachedDocs)
+	} else {
+		indexer.addCacheLock.Unlock()
+	}
+}
+
+// AddDocs 向反向索引表中加入 ADDCACHE 中所有文档
+func (indexer *Indexer) AddDocs(docs *types.DocsIndex) {
+	if indexer.initialized == false {
+		log.Fatal("The Indexer has not been initialized.")
+	}
+
+	indexer.tableLock.Lock()
+	defer indexer.tableLock.Unlock()
+	indexPointers := make(map[string]int, len(indexer.tableLock.table))
+
+	// DocId 递增顺序遍历插入文档保证索引移动次数最少
+	for i, doc := range *docs {
+		if i < len(*docs)-1 && (*docs)[i].DocId == (*docs)[i+1].DocId {
+			// 如果有重复文档加入，因为稳定排序，只加入最后一个
+			continue
+		}
+
+		docState, ok := indexer.tableLock.docsState[doc.DocId]
+		if ok && docState == 1 {
+			// 如果此时 docState 仍为 1，说明该文档需被删除
+			// docState 合法状态为 nil & 2，保证一定不会插入已经在索引表中的文档
+			continue
+		}
+
+		// 更新文档关键词总长度
+		if doc.TokenLen != 0 {
+			indexer.docTokenLens[doc.DocId] = float32(doc.TokenLen)
+			indexer.totalTokenLen += doc.TokenLen
+		}
+
+		docIdIsNew := true
+		for _, keyword := range doc.Keywords {
+			indices, foundKeyword := indexer.tableLock.table[keyword.Text]
+			if !foundKeyword {
+				// 如果没找到该搜索键则加入
+				ti := KeywordIndices{}
+				switch indexer.initOptions.IndexType {
+				case types.LocsIndex:
+					ti.locations = [][]int{keyword.Starts}
+				case types.FrequenciesIndex:
+					ti.frequencies = []float32{keyword.Frequency}
+				}
+				ti.docIds = []uint64{doc.DocId}
+				indexer.tableLock.table[keyword.Text] = &ti
+				continue
+			}
+
+			// 查找应该插入的位置，且索引一定不存在
+			position, _ := indexer.searchIndex(
+				indices, indexPointers[keyword.Text], indexer.getIndexLen(indices)-1, doc.DocId)
+			indexPointers[keyword.Text] = position
+
+			switch indexer.initOptions.IndexType {
+			case types.LocsIndex:
+				indices.locations = append(indices.locations, []int{})
+				copy(indices.locations[position+1:], indices.locations[position:])
+				indices.locations[position] = keyword.Starts
+			case types.FrequenciesIndex:
+				indices.frequencies = append(indices.frequencies, float32(0))
+				copy(indices.frequencies[position+1:], indices.frequencies[position:])
+				indices.frequencies[position] = keyword.Frequency
+			}
+
+			indices.docIds = append(indices.docIds, 0)
+			copy(indices.docIds[position+1:], indices.docIds[position:])
+			indices.docIds[position] = doc.DocId
+		}
+
+		// 更新文章状态和总数
+		if docIdIsNew {
+			indexer.tableLock.docsState[doc.DocId] = 0
+			indexer.numDocs++
+		}
+	}
+}
+
+// RemoveDocToCache 向 REMOVECACHE 中加入一个待删除文档
+// 返回值表示文档是否在索引表中被删除
+func (indexer *Indexer) RemoveDocToCache(docId uint64, forceUpdate bool) bool {
+	if indexer.initialized == false {
+		log.Fatal("The Indexer has not been initialized.")
+	}
+
+	indexer.removeCacheLock.Lock()
+	if docId != 0 {
+		indexer.tableLock.Lock()
+		if docState, ok := indexer.tableLock.docsState[docId]; ok && docState == 0 {
+			indexer.removeCacheLock.removeCache[indexer.removeCacheLock.removeCachePointer] = docId
+			indexer.removeCacheLock.removeCachePointer++
+			indexer.tableLock.docsState[docId] = 1
+			indexer.numDocs--
+		} else if ok && docState == 2 {
+			// 删除一个等待加入的文档
+			indexer.tableLock.docsState[docId] = 1
+		} else if !ok {
+			// 若文档不存在，则无法判断其是否在 addCache 中，需避免这样的操作
+		}
+		indexer.tableLock.Unlock()
+	}
+
+	if indexer.removeCacheLock.removeCachePointer > 0 &&
+		(indexer.removeCacheLock.removeCachePointer >= indexer.initOptions.DocCacheSize ||
+			forceUpdate) {
+		removeCacheddocs := indexer.removeCacheLock.removeCache[:indexer.removeCacheLock.removeCachePointer]
+		indexer.removeCacheLock.removeCachePointer = 0
+		indexer.removeCacheLock.Unlock()
+		sort.Sort(removeCacheddocs)
+		indexer.RemoveDocs(&removeCacheddocs)
+		return true
+	}
+
+	indexer.removeCacheLock.Unlock()
+	return false
+}
+
+// RemoveDocs 向反向索引表中删除 REMOVECACHE 中所有文档
+func (indexer *Indexer) RemoveDocs(docs *types.DocsId) {
+	if indexer.initialized == false {
+		log.Fatal("The Indexer has not been initialized.")
+	}
+
+	indexer.tableLock.Lock()
+	defer indexer.tableLock.Unlock()
+
+	// 更新文档关键词总长度，删除文档状态
+	for _, docId := range *docs {
+		indexer.totalTokenLen -= indexer.docTokenLens[docId]
+		delete(indexer.docTokenLens, docId)
+		delete(indexer.tableLock.docsState, docId)
+	}
+
+	for keyword, indices := range indexer.tableLock.table {
+		indicesTop, indicesPointer := 0, 0
+		docsPointer := sort.Search(
+			len(*docs), func(i int) bool { return (*docs)[i] >= indices.docIds[0] })
+		// 双指针扫描，进行批量删除操作
+		for docsPointer < len(*docs) && indicesPointer < indexer.getIndexLen(indices) {
+			if indices.docIds[indicesPointer] < (*docs)[docsPointer] {
+				if indicesTop != indicesPointer {
+					switch indexer.initOptions.IndexType {
+					case types.LocsIndex:
+						indices.locations[indicesTop] = indices.locations[indicesPointer]
+					case types.FrequenciesIndex:
+						indices.frequencies[indicesTop] = indices.frequencies[indicesPointer]
+					}
+
+					indices.docIds[indicesTop] = indices.docIds[indicesPointer]
+				}
+
+				indicesTop++
+				indicesPointer++
+			} else if indices.docIds[indicesPointer] == (*docs)[docsPointer] {
+				indicesPointer++
+				docsPointer++
+			} else {
+				docsPointer++
+			}
+		}
+		if indicesTop != indicesPointer {
+			switch indexer.initOptions.IndexType {
+			case types.LocsIndex:
+				indices.locations = append(
+					indices.locations[:indicesTop], indices.locations[indicesPointer:]...)
+			case types.FrequenciesIndex:
+				indices.frequencies = append(
+					indices.frequencies[:indicesTop], indices.frequencies[indicesPointer:]...)
+			}
+
+			indices.docIds = append(
+				indices.docIds[:indicesTop], indices.docIds[indicesPointer:]...)
+		}
+
+		if len(indices.docIds) == 0 {
+			delete(indexer.tableLock.table, keyword)
+		}
+	}
+}
+
+// Lookup lookup docs
+// 查找包含全部搜索键(AND操作)的文档
+// 当 docIds 不为 nil 时仅从 docIds 指定的文档中查找
+func (indexer *Indexer) Lookup(
+	tokens []string, labels []string, docIds map[uint64]bool, countDocsOnly bool,
+	logic ...types.Logic) (docs []types.IndexedDoc, numDocs int) {
+
+	if indexer.initialized == false {
+		log.Fatal("The Indexer has not been initialized.")
+	}
+
+	if indexer.numDocs == 0 {
+		return
+	}
+	numDocs = 0
+
+	// 合并关键词和标签为搜索键
+	keywords := make([]string, len(tokens)+len(labels))
+	copy(keywords, tokens)
+	copy(keywords[len(tokens):], labels)
+
+	if len(logic) > 0 {
+		if logic != nil && len(keywords) > 0 && logic[0].Must == true ||
+			logic[0].Should == true || logic[0].NotIn == true {
+
+			docs, numDocs = indexer.LogicLookup(
+				docIds, countDocsOnly, keywords, logic[0])
+
+			return
+		}
+
+		if logic != nil && (len(logic[0].LogicExpr.MustLabels) > 0 ||
+			len(logic[0].LogicExpr.ShouldLabels) > 0) &&
+			len(logic[0].LogicExpr.NotInLabels) >= 0 {
+
+			docs, numDocs = indexer.LogicLookup(
+				docIds, countDocsOnly, keywords, logic[0])
+
+			return
+		}
+	}
+
+	indexer.tableLock.RLock()
+	defer indexer.tableLock.RUnlock()
+
+	table := make([]*KeywordIndices, len(keywords))
+	for i, keyword := range keywords {
+		indices, found := indexer.tableLock.table[keyword]
+		if !found {
+			// 当反向索引表中无此搜索键时直接返回
+			return
+		}
+		// 否则加入反向表中
+		table[i] = indices
+	}
+
+	// 当没有找到时直接返回
+	if len(table) == 0 {
+		return
+	}
+
+	// 归并查找各个搜索键出现文档的交集
+	// 从后向前查保证先输出 DocId 较大文档
+	indexPointers := make([]int, len(table))
+	for iTable := 0; iTable < len(table); iTable++ {
+		indexPointers[iTable] = indexer.getIndexLen(table[iTable]) - 1
+	}
+
+	// 平均文本关键词长度，用于计算BM25
+	avgDocLength := indexer.totalTokenLen / float32(indexer.numDocs)
+	for ; indexPointers[0] >= 0; indexPointers[0]-- {
+		// 以第一个搜索键出现的文档作为基准，并遍历其他搜索键搜索同一文档
+		baseDocId := indexer.getDocId(table[0], indexPointers[0])
+		if docIds != nil {
+			if _, found := docIds[baseDocId]; !found {
+				continue
+			}
+		}
+
+		iTable := 1
+		found := true
+		for ; iTable < len(table); iTable++ {
+			// 二分法比简单的顺序归并效率高，也有更高效率的算法，
+			// 但顺序归并也许是更好的选择，考虑到将来需要用链表重新实现
+			// 以避免反向表添加新文档时的写锁。
+			// TODO: 进一步研究不同求交集算法的速度和可扩展性。
+			position, foundBaseDocId := indexer.searchIndex(table[iTable],
+				0, indexPointers[iTable], baseDocId)
+
+			if foundBaseDocId {
+				indexPointers[iTable] = position
+			} else {
+				if position == 0 {
+					// 该搜索键中所有的文档 ID 都比 baseDocId 大，因此已经没有
+					// 继续查找的必要。
+					return
+				}
+
+				// 继续下一 indexPointers[0] 的查找
+				indexPointers[iTable] = position - 1
+				found = false
+				break
+			}
+		}
+
+		if found {
+			if docState, ok := indexer.tableLock.docsState[baseDocId]; !ok || docState != 0 {
+				continue
+			}
+			indexedDoc := types.IndexedDoc{}
+
+			// 当为 LocsIndex 时计算关键词紧邻距离
+			if indexer.initOptions.IndexType == types.LocsIndex {
+				// 计算有多少关键词是带有距离信息的
+				numTokensWithLocations := 0
+				for i, t := range table[:len(tokens)] {
+					if len(t.locations[indexPointers[i]]) > 0 {
+						numTokensWithLocations++
+					}
+				}
+				if numTokensWithLocations != len(tokens) {
+					if !countDocsOnly {
+						docs = append(docs, types.IndexedDoc{
+							DocId: baseDocId,
+						})
+					}
+					numDocs++
+					//当某个关键字对应多个文档且有 lable 关键字存在时，若直接 break,
+					// 将会丢失相当一部分搜索结果
+					continue
+				}
+
+				// 计算搜索键在文档中的紧邻距离
+				tokenProximity, TokenLocs := computeTokenProximity(
+					table[:len(tokens)], indexPointers, tokens)
+
+				indexedDoc.TokenProximity = int32(tokenProximity)
+				indexedDoc.TokenSnippetLocs = TokenLocs
+
+				// 添加 TokenLocs
+				indexedDoc.TokenLocs = make([][]int, len(tokens))
+				for i, t := range table[:len(tokens)] {
+					indexedDoc.TokenLocs[i] = t.locations[indexPointers[i]]
+				}
+			}
+
+			// 当为 LocsIndex 或者 FrequenciesIndex 时计算BM25
+			if indexer.initOptions.IndexType == types.LocsIndex ||
+				indexer.initOptions.IndexType == types.FrequenciesIndex {
+				bm25 := float32(0)
+				d := indexer.docTokenLens[baseDocId]
+				for i, t := range table[:len(tokens)] {
+					var frequency float32
+					if indexer.initOptions.IndexType == types.LocsIndex {
+						frequency = float32(len(t.locations[indexPointers[i]]))
+					} else {
+						frequency = t.frequencies[indexPointers[i]]
+					}
+
+					// 计算 BM25
+					if len(t.docIds) > 0 && frequency > 0 &&
+						indexer.initOptions.BM25Parameters != nil && avgDocLength != 0 {
+						// 带平滑的 idf
+						idf := float32(math.Log2(float64(indexer.numDocs)/float64(len(t.docIds)) + 1))
+						k1 := indexer.initOptions.BM25Parameters.K1
+						b := indexer.initOptions.BM25Parameters.B
+						bm25 += idf * frequency * (k1 + 1) / (frequency + k1*(1-b+b*d/avgDocLength))
+					}
+				}
+				indexedDoc.BM25 = float32(bm25)
+			}
+
+			indexedDoc.DocId = baseDocId
+			if !countDocsOnly {
+				docs = append(docs, indexedDoc)
+			}
+			numDocs++
+		}
+	}
+
+	return
+}
+
+// searchIndex 二分法查找 indices 中某文档的索引项
+// 第一个返回参数为找到的位置或需要插入的位置
+// 第二个返回参数标明是否找到
+func (indexer *Indexer) searchIndex(indices *KeywordIndices,
+	start int, end int, docId uint64) (int, bool) {
+	// 特殊情况
+	if indexer.getIndexLen(indices) == start {
+		return start, false
+	}
+	if docId < indexer.getDocId(indices, start) {
+		return start, false
+	} else if docId == indexer.getDocId(indices, start) {
+		return start, true
+	}
+	if docId > indexer.getDocId(indices, end) {
+		return end + 1, false
+	} else if docId == indexer.getDocId(indices, end) {
+		return end, true
+	}
+
+	// 二分
+	var middle int
+	for end-start > 1 {
+		middle = (start + end) / 2
+		if docId == indexer.getDocId(indices, middle) {
+			return middle, true
+		} else if docId > indexer.getDocId(indices, middle) {
+			start = middle
+		} else {
+			end = middle
+		}
+	}
+
+	return end, false
+}
+
+// computeTokenProximity 计算搜索键在文本中的紧邻距离
+//
+// 假定第 i 个搜索键首字节出现在文本中的位置为 P_i，长度 L_i
+// 紧邻距离计算公式为
+//
+// 	ArgMin(Sum(Abs(P_(i+1) - P_i - L_i)))
+//
+// 具体由动态规划实现，依次计算前 i 个 token 在每个出现位置的最优值。
+// 选定的 P_i 通过 TokenLocs 参数传回。
+func computeTokenProximity(table []*KeywordIndices,
+	indexPointers []int, tokens []string) (
+	minTokenProximity int, TokenLocs []int) {
+	minTokenProximity = -1
+	TokenLocs = make([]int, len(tokens))
+
+	var (
+		currentLocations, nextLocations []int
+		currentMinValues, nextMinValues []int
+		path                            [][]int
+	)
+
+	// 初始化路径数组
+	path = make([][]int, len(tokens))
+	for i := 1; i < len(path); i++ {
+		path[i] = make([]int, len(table[i].locations[indexPointers[i]]))
+	}
+
+	// 动态规划
+	currentLocations = table[0].locations[indexPointers[0]]
+	currentMinValues = make([]int, len(currentLocations))
+	for i := 1; i < len(tokens); i++ {
+		nextLocations = table[i].locations[indexPointers[i]]
+		nextMinValues = make([]int, len(nextLocations))
+		for j := range nextMinValues {
+			nextMinValues[j] = -1
+		}
+
+		var iNext int
+		for iCurrent, currentLocation := range currentLocations {
+			if currentMinValues[iCurrent] == -1 {
+				continue
+			}
+			for iNext+1 < len(nextLocations) &&
+				nextLocations[iNext+1] < currentLocation {
+				iNext++
+			}
+
+			update := func(from int, to int) {
+				if to >= len(nextLocations) {
+					return
+				}
+				value := currentMinValues[from] +
+					utils.AbsInt(nextLocations[to]-currentLocations[from]-len(tokens[i-1]))
+
+				if nextMinValues[to] == -1 || value < nextMinValues[to] {
+					nextMinValues[to] = value
+					path[i][to] = from
+				}
+			}
+
+			// 最优解的状态转移只发生在左右最接近的位置
+			update(iCurrent, iNext)
+			update(iCurrent, iNext+1)
+		}
+
+		currentLocations = nextLocations
+		currentMinValues = nextMinValues
+	}
+
+	// 找出最优解
+	var cursor int
+	for i, value := range currentMinValues {
+		if value == -1 {
+			continue
+		}
+		if minTokenProximity == -1 || value < minTokenProximity {
+			minTokenProximity = value
+			cursor = i
+		}
+	}
+
+	// 从路径倒推出最优解的位置
+	for i := len(tokens) - 1; i >= 0; i-- {
+		if i != len(tokens)-1 {
+			cursor = path[i+1][cursor]
+		}
+		TokenLocs[i] = table[i].locations[indexPointers[i]][cursor]
+	}
+
+	return
+}
+
+// LogicLookup logic Lookup
+func (indexer *Indexer) LogicLookup(
+	docIds map[uint64]bool, countDocsOnly bool, LogicExpr []string,
+	logic types.Logic) (docs []types.IndexedDoc, numDocs int) {
+
+	indexer.tableLock.RLock()
+	defer indexer.tableLock.RUnlock()
+
+	// // 有效性检查, 不允许只出现逻辑非检索, 也不允许与或非都不存在
+	// if Logic.Must == true && Logic.Should == true && Logic.NotIn == true {
+	// 	return
+	// }
+
+	// MustTable 中的搜索键检查
+	// 如果存在与搜索键, 则要求所有的与搜索键都有对应的反向表
+	MustTable := make([]*KeywordIndices, 0)
+
+	if len(logic.LogicExpr.MustLabels) > 0 {
+		LogicExpr = logic.LogicExpr.MustLabels
+	}
+	if logic.Must == true || len(logic.LogicExpr.MustLabels) > 0 {
+		for _, keyword := range LogicExpr {
+			indices, found := indexer.tableLock.table[keyword]
+			if !found {
+				return
+			}
+
+			MustTable = append(MustTable, indices)
+		}
+	}
+
+	// 逻辑或搜索键检查
+	// 1. 如果存在逻辑或搜索键, 则至少有一个存在反向表
+	// 2. 逻辑或和逻辑与之间是与关系
+	ShouldTable := make([]*KeywordIndices, 0)
+
+	if len(logic.LogicExpr.ShouldLabels) > 0 {
+		LogicExpr = logic.LogicExpr.ShouldLabels
+	}
+
+	if logic.Should == true || len(logic.LogicExpr.ShouldLabels) > 0 {
+		for _, keyword := range LogicExpr {
+			indices, found := indexer.tableLock.table[keyword]
+			if found {
+				ShouldTable = append(ShouldTable, indices)
+			}
+		}
+		if len(ShouldTable) == 0 {
+			// 如果存在逻辑或搜索键， 但是对应的反向表全部为空， 则返回
+			return
+		}
+	}
+
+	// 逻辑非中的搜索键检查
+	// 可以不存在逻辑非搜索（NotInTable为空), 允许逻辑非搜索键对应的反向表为空
+	NotInTable := make([]*KeywordIndices, 0)
+
+	if len(logic.LogicExpr.NotInLabels) > 0 {
+		LogicExpr = logic.LogicExpr.NotInLabels
+	}
+	if logic.NotIn == true || len(logic.LogicExpr.NotInLabels) > 0 {
+		for _, keyword := range LogicExpr {
+			indices, found := indexer.tableLock.table[keyword]
+			if found {
+				NotInTable = append(NotInTable, indices)
+			}
+		}
+	}
+
+	// 开始检索
+	numDocs = 0
+	if logic.Must == true || len(logic.LogicExpr.MustLabels) > 0 {
+		// 如果存在逻辑与检索
+		for idx := indexer.getIndexLen(MustTable[0]) - 1; idx >= 0; idx-- {
+			baseDocId := indexer.getDocId(MustTable[0], idx)
+			if docIds != nil {
+				_, found := docIds[baseDocId]
+				if !found {
+					continue
+				}
+			}
+
+			mustFound := indexer.findInMustTable(MustTable[1:], baseDocId)
+			shouldFound := indexer.findInShouldTable(ShouldTable, baseDocId)
+			notInFound := indexer.findInNotInTable(NotInTable, baseDocId)
+
+			if mustFound && shouldFound && !notInFound {
+				indexedDoc := types.IndexedDoc{}
+				indexedDoc.DocId = baseDocId
+				if !countDocsOnly {
+					docs = append(docs, indexedDoc)
+				}
+				numDocs++
+			}
+		}
+	} else {
+		// 不存在逻辑与检索, 则必须存在逻辑或检索
+		// 这时进行求并集操作
+		if logic.Should == true || len(logic.LogicExpr.ShouldLabels) > 0 {
+			docs, numDocs = indexer.unionTable(ShouldTable, NotInTable, countDocsOnly)
+		} else {
+			uintDocIds := make([]uint64, 0)
+			// 当前直接返回 Not 逻辑数据
+			for i := 0; i < len(NotInTable); i++ {
+				for _, docid := range NotInTable[i].docIds {
+					if indexer.findInNotInTable(NotInTable, docid) {
+						uintDocIds = append(uintDocIds, docid)
+					}
+				}
+			}
+
+			StableDesc(uintDocIds)
+
+			numDocs = 0
+			for _, doc := range uintDocIds {
+				indexedDoc := types.IndexedDoc{}
+				indexedDoc.DocId = doc
+				if !countDocsOnly {
+					docs = append(docs, indexedDoc)
+				}
+				numDocs++
+			}
+		}
+
+		// fmt.Println(docs, numDocs)
+	}
+
+	return
+}
+
+// 在逻辑与反向表中对docid进行查找, 若每个反向表都找到,
+// 则返回 true, 有一个找不到则返回 false
+func (indexer *Indexer) findInMustTable(table []*KeywordIndices, docId uint64) bool {
+	for i := 0; i < len(table); i++ {
+		_, foundDocId := indexer.searchIndex(table[i],
+			0, indexer.getIndexLen(table[i])-1, docId)
+		if !foundDocId {
+			return false
+		}
+	}
+
+	return true
+}
+
+// 在逻辑或反向表中对 docid 进行查找， 若有一个找到则返回 true,
+// 都找不到则返回 false
+// 如果 table 为空， 则返回 true
+func (indexer *Indexer) findInShouldTable(table []*KeywordIndices, docId uint64) bool {
+	for i := 0; i < len(table); i++ {
+		_, foundDocId := indexer.searchIndex(table[i],
+			0, indexer.getIndexLen(table[i])-1, docId)
+		if foundDocId {
+			return true
+		}
+	}
+
+	if len(table) == 0 {
+		return true
+	}
+
+	return false
+}
+
+// findInNotInTable 在逻辑非反向表中对 docid 进行查找,
+// 若有一个找到则返回 true, 都找不到则返回 false
+// 如果 table 为空, 则返回 false
+func (indexer *Indexer) findInNotInTable(table []*KeywordIndices, docId uint64) bool {
+	for i := 0; i < len(table); i++ {
+		_, foundDocId := indexer.searchIndex(table[i],
+			0, indexer.getIndexLen(table[i])-1, docId)
+		if foundDocId {
+			return true
+		}
+	}
+
+	return false
+}
+
+// unionTable 如果不存在与逻辑检索， 则需要对逻辑或反向表求并集
+// 先求差集再求并集， 可以减小内存占用
+// docid 要保序
+func (indexer *Indexer) unionTable(table []*KeywordIndices,
+	notInTable []*KeywordIndices, countDocsOnly bool) (
+	docs []types.IndexedDoc, numDocs int) {
+	docIds := make([]uint64, 0)
+	// 求并集
+	for i := 0; i < len(table); i++ {
+		for _, docid := range table[i].docIds {
+			if !indexer.findInNotInTable(notInTable, docid) {
+				found := false
+				for _, v := range docIds {
+					if v == docid {
+						found = true
+						break
+					}
+				}
+				if !found {
+					docIds = append(docIds, docid)
+				}
+			}
+		}
+	}
+	// 排序
+	// sortUint64.StableDesc(docIds)
+	StableDesc(docIds)
+
+	numDocs = 0
+	for _, doc := range docIds {
+		indexedDoc := types.IndexedDoc{}
+		indexedDoc.DocId = doc
+		if !countDocsOnly {
+			docs = append(docs, indexedDoc)
+		}
+		numDocs++
+	}
+
+	return
+}

vendor/github.com/go-ego/riot/core/ranker.go

@@ -0,0 +1,243 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package core
+
+import (
+	// "fmt"
+	"log"
+	"sort"
+	"sync"
+
+	"github.com/go-ego/riot/types"
+	"github.com/go-ego/riot/utils"
+)
+
+// Ranker ranker
+type Ranker struct {
+	idOnly bool
+
+	lock struct {
+		sync.RWMutex
+		fields map[uint64]interface{}
+		docs   map[uint64]bool
+		// new
+		content map[uint64]string
+		attri   map[uint64]interface{}
+	}
+
+	initialized bool
+}
+
+// Init init ranker
+func (ranker *Ranker) Init(onlyID ...bool) {
+	if ranker.initialized == true {
+		log.Fatal("The Ranker can not be initialized twice.")
+	}
+	ranker.initialized = true
+
+	ranker.lock.fields = make(map[uint64]interface{})
+	ranker.lock.docs = make(map[uint64]bool)
+
+	if len(onlyID) > 0 {
+		ranker.idOnly = onlyID[0]
+	}
+
+	if !ranker.idOnly {
+		// new
+		ranker.lock.content = make(map[uint64]string)
+		ranker.lock.attri = make(map[uint64]interface{})
+	}
+}
+
+// AddDoc add doc
+// 给某个文档添加评分字段
+func (ranker *Ranker) AddDoc(
+	// docId uint64, fields interface{}, content string, attri interface{}) {
+	docId uint64, fields interface{}, content ...interface{}) {
+	if ranker.initialized == false {
+		log.Fatal("The Ranker has not been initialized.")
+	}
+
+	ranker.lock.Lock()
+	ranker.lock.fields[docId] = fields
+	ranker.lock.docs[docId] = true
+
+	if !ranker.idOnly {
+		// new
+		if len(content) > 0 {
+			ranker.lock.content[docId] = content[0].(string)
+		}
+
+		if len(content) > 1 {
+			ranker.lock.attri[docId] = content[1]
+			// ranker.lock.attri[docId] = attri
+		}
+	}
+
+	ranker.lock.Unlock()
+}
+
+// RemoveDoc 删除某个文档的评分字段
+func (ranker *Ranker) RemoveDoc(docId uint64) {
+	if ranker.initialized == false {
+		log.Fatal("The Ranker has not been initialized.")
+	}
+
+	ranker.lock.Lock()
+	delete(ranker.lock.fields, docId)
+	delete(ranker.lock.docs, docId)
+
+	if !ranker.idOnly {
+		// new
+		delete(ranker.lock.content, docId)
+		delete(ranker.lock.attri, docId)
+	}
+
+	ranker.lock.Unlock()
+}
+
+// RankDocId rank docs by types.ScoredIDs
+func (ranker *Ranker) RankDocId(docs []types.IndexedDoc,
+	options types.RankOpts, countDocsOnly bool) (types.ScoredIDs, int) {
+	var outputDocs types.ScoredIDs
+	numDocs := 0
+
+	for _, d := range docs {
+		ranker.lock.RLock()
+		// 判断 doc 是否存在
+		if _, ok := ranker.lock.docs[d.DocId]; ok {
+
+			fs := ranker.lock.fields[d.DocId]
+
+			ranker.lock.RUnlock()
+			// 计算评分并剔除没有分值的文档
+			scores := options.ScoringCriteria.Score(d, fs)
+			if len(scores) > 0 {
+				if !countDocsOnly {
+					outputDocs = append(outputDocs, types.ScoredID{
+						DocId:            d.DocId,
+						Scores:           scores,
+						TokenSnippetLocs: d.TokenSnippetLocs,
+						TokenLocs:        d.TokenLocs})
+				}
+				numDocs++
+			}
+		} else {
+			ranker.lock.RUnlock()
+		}
+	}
+
+	// 排序
+	if !countDocsOnly {
+		if options.ReverseOrder {
+			sort.Sort(sort.Reverse(outputDocs))
+		} else {
+			sort.Sort(outputDocs)
+		}
+		// 当用户要求只返回部分结果时返回部分结果
+		var start, end int
+		if options.MaxOutputs != 0 {
+			start = utils.MinInt(options.OutputOffset, len(outputDocs))
+			end = utils.MinInt(options.OutputOffset+options.MaxOutputs, len(outputDocs))
+		} else {
+			start = utils.MinInt(options.OutputOffset, len(outputDocs))
+			end = len(outputDocs)
+		}
+		return outputDocs[start:end], numDocs
+	}
+
+	return outputDocs, numDocs
+}
+
+// RankDocs rank docs by types.ScoredDocs
+func (ranker *Ranker) RankDocs(docs []types.IndexedDoc,
+	options types.RankOpts, countDocsOnly bool) (types.ScoredDocs, int) {
+	var outputDocs types.ScoredDocs
+	numDocs := 0
+
+	for _, d := range docs {
+		ranker.lock.RLock()
+		// 判断 doc 是否存在
+		if _, ok := ranker.lock.docs[d.DocId]; ok {
+
+			fs := ranker.lock.fields[d.DocId]
+			content := ranker.lock.content[d.DocId]
+			attri := ranker.lock.attri[d.DocId]
+
+			ranker.lock.RUnlock()
+			// 计算评分并剔除没有分值的文档
+			scores := options.ScoringCriteria.Score(d, fs)
+			if len(scores) > 0 {
+				if !countDocsOnly {
+					outputDocs = append(outputDocs, types.ScoredDoc{
+						DocId: d.DocId,
+						// new
+						Fields:  fs,
+						Content: content,
+						Attri:   attri,
+						//
+						Scores:           scores,
+						TokenSnippetLocs: d.TokenSnippetLocs,
+						TokenLocs:        d.TokenLocs})
+				}
+				numDocs++
+			}
+		} else {
+			ranker.lock.RUnlock()
+		}
+	}
+
+	// 排序
+	if !countDocsOnly {
+		if options.ReverseOrder {
+			sort.Sort(sort.Reverse(outputDocs))
+		} else {
+			sort.Sort(outputDocs)
+		}
+		// 当用户要求只返回部分结果时返回部分结果
+		var start, end int
+		if options.MaxOutputs != 0 {
+			start = utils.MinInt(options.OutputOffset, len(outputDocs))
+			end = utils.MinInt(options.OutputOffset+options.MaxOutputs, len(outputDocs))
+		} else {
+			start = utils.MinInt(options.OutputOffset, len(outputDocs))
+			end = len(outputDocs)
+		}
+		return outputDocs[start:end], numDocs
+	}
+
+	return outputDocs, numDocs
+}
+
+// Rank rank docs
+// 给文档评分并排序
+func (ranker *Ranker) Rank(docs []types.IndexedDoc,
+	options types.RankOpts, countDocsOnly bool) (
+	interface{}, int) {
+
+	if ranker.initialized == false {
+		log.Fatal("The Ranker has not been initialized.")
+	}
+
+	// 对每个文档评分
+	if ranker.idOnly {
+		outputDocs, numDocs := ranker.RankDocId(docs, options, countDocsOnly)
+		return outputDocs, numDocs
+	}
+
+	outputDocs, numDocs := ranker.RankDocs(docs, options, countDocsOnly)
+	return outputDocs, numDocs
+}

vendor/github.com/go-ego/riot/core/test_utils.go

@@ -0,0 +1,44 @@
+package core
+
+import (
+	"fmt"
+
+	"github.com/go-ego/riot/types"
+)
+
+func indicesToString(indexer *Indexer, token string) (output string) {
+	if indices, ok := indexer.tableLock.table[token]; ok {
+		for i := 0; i < indexer.getIndexLen(indices); i++ {
+			output += fmt.Sprintf("%d ",
+				indexer.getDocId(indices, i))
+		}
+	}
+	return
+}
+
+func indexedDocsToString(docs []types.IndexedDoc, numDocs int) (output string) {
+	for _, doc := range docs {
+		output += fmt.Sprintf("[%d %d %v] ",
+			doc.DocId, doc.TokenProximity, doc.TokenSnippetLocs)
+	}
+	return
+}
+
+func scoredDocsToString(docs []types.ScoredDoc) (output string) {
+	for _, doc := range docs {
+		output += fmt.Sprintf("[%d [", doc.DocId)
+		for _, score := range doc.Scores {
+			output += fmt.Sprintf("%d ", int(score*1000))
+		}
+		output += "]] "
+	}
+	return
+}
+
+func indexedDocIdsToString(docs []types.IndexedDoc, numDocs int) (output string) {
+	for _, doc := range docs {
+		output += fmt.Sprintf("[%d] ",
+			doc.DocId)
+	}
+	return
+}

vendor/github.com/go-ego/riot/core/uint64.go

@@ -0,0 +1,496 @@
+package core
+
+// ================= COMMON =================
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+// ------------- ASCENDING -------------
+
+func heapSortAsc(data []uint64, a, b int) {
+	first := a
+	lo := 0
+	hi := b - a
+	for i := (hi - 1) / 2; i >= 0; i-- {
+		siftDownAsc(data, i, hi, first)
+	}
+	for i := hi - 1; i >= 0; i-- {
+		data[first], data[first+i] = data[first+i], data[first]
+		siftDownAsc(data, lo, i, first)
+	}
+}
+
+func insertionSortAsc(data []uint64, a, b int) {
+	var j int
+	for i := a + 1; i < b; i++ {
+		for j = i; j > a && data[j] < data[j-1]; j-- {
+			data[j], data[j-1] = data[j-1], data[j]
+		}
+	}
+}
+
+func siftDownAsc(data []uint64, lo, hi, first int) {
+	root := lo
+	for {
+		child := 2*root + 1
+		if child >= hi {
+			break
+		}
+		if child+1 < hi && data[first+child] < data[first+child+1] {
+			child++
+		}
+		if data[first+root] >= data[first+child] {
+			return
+		}
+		data[first+root], data[first+child] = data[first+child], data[first+root]
+		root = child
+	}
+}
+
+func medianOfThreeAsc(data []uint64, m1, m0, m2 int) {
+	// bubble sort on 3 elements
+	if data[m1] < data[m0] {
+		data[m1], data[m0] = data[m0], data[m1]
+	}
+	if data[m2] < data[m1] {
+		data[m2], data[m1] = data[m1], data[m2]
+	}
+	if data[m1] < data[m0] {
+		data[m1], data[m0] = data[m0], data[m1]
+	}
+}
+
+func swapRangeAsc(data []uint64, a, b, n int) {
+	for i := 0; i < n; i++ {
+		data[a], data[b] = data[b], data[a]
+		a++
+		b++
+	}
+}
+
+func doPivotAsc(data []uint64, lo, hi int) (midlo, midhi int) {
+	m := lo + (hi-lo)/2
+	if hi-lo > 40 {
+		s := (hi - lo) / 8
+		medianOfThreeAsc(data, lo, lo+s, lo+2*s)
+		medianOfThreeAsc(data, m, m-s, m+s)
+		medianOfThreeAsc(data, hi-1, hi-1-s, hi-1-2*s)
+	}
+	medianOfThreeAsc(data, lo, m, hi-1)
+
+	pivot := lo
+	a, b, c, d := lo+1, lo+1, hi, hi
+	for {
+		for b < c {
+			if data[b] < data[pivot] {
+				b++
+			} else if data[pivot] >= data[b] {
+				data[a], data[b] = data[b], data[a]
+				a++
+				b++
+			} else {
+				break
+			}
+		}
+		for b < c {
+			if data[pivot] < data[c-1] {
+				c--
+			} else if data[c-1] >= data[pivot] {
+				data[c-1], data[d-1] = data[d-1], data[c-1]
+				c--
+				d--
+			} else {
+				break
+			}
+		}
+		if b >= c {
+			break
+		}
+		data[b], data[c-1] = data[c-1], data[b]
+		b++
+		c--
+	}
+
+	n := min(b-a, a-lo)
+	swapRangeAsc(data, lo, b-n, n)
+
+	n = min(hi-d, d-c)
+	swapRangeAsc(data, c, hi-n, n)
+
+	return lo + b - a, hi - (d - c)
+}
+
+func quickSortAsc(data []uint64, a, b, maxDepth int) {
+	var mlo, mhi int
+	for b-a > 7 {
+		if maxDepth == 0 {
+			heapSortAsc(data, a, b)
+			return
+		}
+		maxDepth--
+		mlo, mhi = doPivotAsc(data, a, b)
+		if mlo-a < b-mhi {
+			quickSortAsc(data, a, mlo, maxDepth)
+			a = mhi
+		} else {
+			quickSortAsc(data, mhi, b, maxDepth)
+			b = mlo
+		}
+	}
+	if b-a > 1 {
+		insertionSortAsc(data, a, b)
+	}
+}
+
+// Asc asc
+func Asc(data []uint64) {
+	maxDepth := 0
+	for i := len(data); i > 0; i >>= 1 {
+		maxDepth++
+	}
+	maxDepth *= 2
+	quickSortAsc(data, 0, len(data), maxDepth)
+}
+
+// IsSortedAsc sorted by Asc
+func IsSortedAsc(data []uint64) bool {
+	for i := len(data) - 1; i > 0; i-- {
+		if data[i] < data[i-1] {
+			return false
+		}
+	}
+	return true
+}
+
+// StableAsc stable Asc
+func StableAsc(data []uint64) {
+	n := len(data)
+	blockSize := 20
+	a, b := 0, blockSize
+	for b <= n {
+		insertionSortAsc(data, a, b)
+		a = b
+		b += blockSize
+	}
+	insertionSortAsc(data, a, n)
+
+	for blockSize < n {
+		a, b = 0, 2*blockSize
+		for b <= n {
+			symMergeAsc(data, a, a+blockSize, b)
+			a = b
+			b += 2 * blockSize
+		}
+		symMergeAsc(data, a, a+blockSize, n)
+		blockSize *= 2
+	}
+}
+
+func symMergeAsc(data []uint64, a, m, b int) {
+	if a >= m || m >= b {
+		return
+	}
+	mid := a + (b-a)/2
+	n := mid + m
+	var start, c, r, p int
+	if m > mid {
+		start = n - b
+		r, p = mid, n-1
+		for start < r {
+			c = start + (r-start)/2
+			if data[p-c] >= data[c] {
+				start = c + 1
+			} else {
+				r = c
+			}
+		}
+	} else {
+		start = a
+		r, p = m, n-1
+		for start < r {
+			c = start + (r-start)/2
+			if data[p-c] >= data[c] {
+				start = c + 1
+			} else {
+				r = c
+			}
+		}
+	}
+	end := n - start
+	rotateAsc(data, start, m, end)
+	symMergeAsc(data, a, start, mid)
+	symMergeAsc(data, mid, end, b)
+}
+
+func rotateAsc(data []uint64, a, m, b int) {
+	i := m - a
+	if i == 0 {
+		return
+	}
+	j := b - m
+	if j == 0 {
+		return
+	}
+	if i == j {
+		swapRangeAsc(data, a, m, i)
+		return
+	}
+	p := a + i
+	for i != j {
+		if i > j {
+			swapRangeAsc(data, p-i, p, j)
+			i -= j
+		} else {
+			swapRangeAsc(data, p-i, p+j-i, i)
+			j -= i
+		}
+	}
+	swapRangeAsc(data, p-i, p, i)
+}
+
+// ------------- DESCENDING -------------
+
+func heapSortDesc(data []uint64, a, b int) {
+	first := a
+	lo := 0
+	hi := b - a
+	for i := (hi - 1) / 2; i >= 0; i-- {
+		siftDownDesc(data, i, hi, first)
+	}
+	for i := hi - 1; i >= 0; i-- {
+		data[first], data[first+i] = data[first+i], data[first]
+		siftDownDesc(data, lo, i, first)
+	}
+}
+
+func insertionSortDesc(data []uint64, a, b int) {
+	var j int
+	for i := a + 1; i < b; i++ {
+		for j = i; j > a && data[j] > data[j-1]; j-- {
+			data[j], data[j-1] = data[j-1], data[j]
+		}
+	}
+}
+
+func siftDownDesc(data []uint64, lo, hi, first int) {
+	root := lo
+	for {
+		child := 2*root + 1
+		if child >= hi {
+			break
+		}
+		if child+1 < hi && data[first+child] > data[first+child+1] {
+			child++
+		}
+		if data[first+root] <= data[first+child] {
+			return
+		}
+		data[first+root], data[first+child] = data[first+child], data[first+root]
+		root = child
+	}
+}
+
+func medianOfThreeDesc(data []uint64, m1, m0, m2 int) {
+	// bubble sort on 3 elements
+	if data[m1] > data[m0] {
+		data[m1], data[m0] = data[m0], data[m1]
+	}
+	if data[m2] > data[m1] {
+		data[m2], data[m1] = data[m1], data[m2]
+	}
+	if data[m1] > data[m0] {
+		data[m1], data[m0] = data[m0], data[m1]
+	}
+}
+
+func swapRangeDesc(data []uint64, a, b, n int) {
+	for i := 0; i < n; i++ {
+		data[a], data[b] = data[b], data[a]
+		a++
+		b++
+	}
+}
+
+func doPivotDesc(data []uint64, lo, hi int) (midlo, midhi int) {
+	m := lo + (hi-lo)/2
+	if hi-lo > 40 {
+		s := (hi - lo) / 8
+		medianOfThreeDesc(data, lo, lo+s, lo+2*s)
+		medianOfThreeDesc(data, m, m-s, m+s)
+		medianOfThreeDesc(data, hi-1, hi-1-s, hi-1-2*s)
+	}
+	medianOfThreeDesc(data, lo, m, hi-1)
+
+	pivot := lo
+	a, b, c, d := lo+1, lo+1, hi, hi
+	for {
+		for b < c {
+			if data[b] > data[pivot] {
+				b++
+			} else if data[pivot] <= data[b] {
+				data[a], data[b] = data[b], data[a]
+				a++
+				b++
+			} else {
+				break
+			}
+		}
+		for b < c {
+			if data[pivot] > data[c-1] {
+				c--
+			} else if data[c-1] <= data[pivot] {
+				data[c-1], data[d-1] = data[d-1], data[c-1]
+				c--
+				d--
+			} else {
+				break
+			}
+		}
+		if b >= c {
+			break
+		}
+		data[b], data[c-1] = data[c-1], data[b]
+		b++
+		c--
+	}
+
+	n := min(b-a, a-lo)
+	swapRangeDesc(data, lo, b-n, n)
+
+	n = min(hi-d, d-c)
+	swapRangeDesc(data, c, hi-n, n)
+
+	return lo + b - a, hi - (d - c)
+}
+
+func quickSortDesc(data []uint64, a, b, maxDepth int) {
+	var mlo, mhi int
+	for b-a > 7 {
+		if maxDepth == 0 {
+			heapSortDesc(data, a, b)
+			return
+		}
+		maxDepth--
+		mlo, mhi = doPivotDesc(data, a, b)
+		if mlo-a < b-mhi {
+			quickSortDesc(data, a, mlo, maxDepth)
+			a = mhi
+		} else {
+			quickSortDesc(data, mhi, b, maxDepth)
+			b = mlo
+		}
+	}
+	if b-a > 1 {
+		insertionSortDesc(data, a, b)
+	}
+}
+
+// Desc desc
+func Desc(data []uint64) {
+	maxDepth := 0
+	for i := len(data); i > 0; i >>= 1 {
+		maxDepth++
+	}
+	maxDepth *= 2
+	quickSortDesc(data, 0, len(data), maxDepth)
+}
+
+// IsSortedDesc  sorted by Desc
+func IsSortedDesc(data []uint64) bool {
+	for i := len(data) - 1; i > 0; i-- {
+		if data[i] > data[i-1] {
+			return false
+		}
+	}
+	return true
+}
+
+// StableDesc stable desc
+func StableDesc(data []uint64) {
+	n := len(data)
+	blockSize := 20
+	a, b := 0, blockSize
+	for b <= n {
+		insertionSortDesc(data, a, b)
+		a = b
+		b += blockSize
+	}
+	insertionSortDesc(data, a, n)
+
+	for blockSize < n {
+		a, b = 0, 2*blockSize
+		for b <= n {
+			symMergeDesc(data, a, a+blockSize, b)
+			a = b
+			b += 2 * blockSize
+		}
+		symMergeDesc(data, a, a+blockSize, n)
+		blockSize *= 2
+	}
+}
+
+func symMergeDesc(data []uint64, a, m, b int) {
+	if a >= m || m >= b {
+		return
+	}
+	mid := a + (b-a)/2
+	n := mid + m
+	var start, c, r, p int
+	if m > mid {
+		start = n - b
+		r, p = mid, n-1
+		for start < r {
+			c = start + (r-start)/2
+			if data[p-c] < data[c] {
+				start = c + 1
+			} else {
+				r = c
+			}
+		}
+	} else {
+		start = a
+		r, p = m, n-1
+		for start < r {
+			c = start + (r-start)/2
+			if data[p-c] < data[c] {
+				start = c + 1
+			} else {
+				r = c
+			}
+		}
+	}
+	end := n - start
+	rotateDesc(data, start, m, end)
+	symMergeDesc(data, a, start, mid)
+	symMergeDesc(data, mid, end, b)
+}
+
+func rotateDesc(data []uint64, a, m, b int) {
+	i := m - a
+	if i == 0 {
+		return
+	}
+	j := b - m
+	if j == 0 {
+		return
+	}
+	if i == j {
+		swapRangeDesc(data, a, m, i)
+		return
+	}
+	p := a + i
+	for i != j {
+		if i > j {
+			swapRangeDesc(data, p-i, p, j)
+			i -= j
+		} else {
+			swapRangeDesc(data, p-i, p+j-i, i)
+			j -= i
+		}
+	}
+	swapRangeDesc(data, p-i, p, i)
+}

vendor/github.com/go-ego/riot/counters.go

@@ -0,0 +1,31 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package riot
+
+// NumTokenIndexAdded added token index number
+func (engine *Engine) NumTokenIndexAdded() uint64 {
+	return engine.numTokenIndexAdded
+}
+
+// NumDocsIndexed documents indexed number
+func (engine *Engine) NumDocsIndexed() uint64 {
+	return engine.numDocsIndexed
+}
+
+// NumDocsRemoved documents removed number
+func (engine *Engine) NumDocsRemoved() uint64 {
+	return engine.numDocsRemoved
+}

vendor/github.com/go-ego/riot/engine.go

@@ -0,0 +1,790 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+/*
+
+Package riot is riot engine
+*/
+package riot
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+	// "reflect"
+
+	"sync/atomic"
+
+	"github.com/go-ego/riot/core"
+	"github.com/go-ego/riot/store"
+	"github.com/go-ego/riot/types"
+	"github.com/go-ego/riot/utils"
+
+	"github.com/go-ego/gse"
+	"github.com/go-ego/murmur"
+	"github.com/shirou/gopsutil/mem"
+)
+
+const (
+	// Version get the riot version
+	Version string = "v0.10.0.425, Danube River!"
+
+	// NumNanosecondsInAMillisecond nano-seconds in a milli-second num
+	NumNanosecondsInAMillisecond = 1000000
+	// StoreFilePrefix persistent store file prefix
+	StoreFilePrefix = "riot"
+
+	// DefaultPath default db path
+	DefaultPath = "./riot-index"
+)
+
+// GetVersion get the riot version
+func GetVersion() string {
+	return Version
+}
+
+// Engine initialize the engine
+type Engine struct {
+	loc sync.RWMutex
+
+	// 计数器，用来统计有多少文档被索引等信息
+	numDocsIndexed       uint64
+	numDocsRemoved       uint64
+	numDocsForceUpdated  uint64
+	numIndexingReqs      uint64
+	numRemovingReqs      uint64
+	numForceUpdatingReqs uint64
+	numTokenIndexAdded   uint64
+	numDocsStored        uint64
+
+	// 记录初始化参数
+	initOptions types.EngineOpts
+	initialized bool
+
+	indexers   []core.Indexer
+	rankers    []core.Ranker
+	segmenter  gse.Segmenter
+	loaded     bool
+	stopTokens StopTokens
+	dbs        []store.Store
+
+	// 建立索引器使用的通信通道
+	segmenterChan         chan segmenterReq
+	indexerAddDocChans    []chan indexerAddDocReq
+	indexerRemoveDocChans []chan indexerRemoveDocReq
+	rankerAddDocChans     []chan rankerAddDocReq
+
+	// 建立排序器使用的通信通道
+	indexerLookupChans   []chan indexerLookupReq
+	rankerRankChans      []chan rankerRankReq
+	rankerRemoveDocChans []chan rankerRemoveDocReq
+
+	// 建立持久存储使用的通信通道
+	storeIndexDocChans []chan storeIndexDocReq
+	storeInitChan      chan bool
+}
+
+// Indexer initialize the indexer channel
+func (engine *Engine) Indexer(options types.EngineOpts) {
+	engine.indexerAddDocChans = make(
+		[]chan indexerAddDocReq, options.NumShards)
+
+	engine.indexerRemoveDocChans = make(
+		[]chan indexerRemoveDocReq, options.NumShards)
+
+	engine.indexerLookupChans = make(
+		[]chan indexerLookupReq, options.NumShards)
+
+	for shard := 0; shard < options.NumShards; shard++ {
+		engine.indexerAddDocChans[shard] = make(
+			chan indexerAddDocReq, options.IndexerBufLen)
+
+		engine.indexerRemoveDocChans[shard] = make(
+			chan indexerRemoveDocReq, options.IndexerBufLen)
+
+		engine.indexerLookupChans[shard] = make(
+			chan indexerLookupReq, options.IndexerBufLen)
+	}
+}
+
+// Ranker initialize the ranker channel
+func (engine *Engine) Ranker(options types.EngineOpts) {
+	engine.rankerAddDocChans = make(
+		[]chan rankerAddDocReq, options.NumShards)
+
+	engine.rankerRankChans = make(
+		[]chan rankerRankReq, options.NumShards)
+
+	engine.rankerRemoveDocChans = make(
+		[]chan rankerRemoveDocReq, options.NumShards)
+
+	for shard := 0; shard < options.NumShards; shard++ {
+		engine.rankerAddDocChans[shard] = make(
+			chan rankerAddDocReq, options.RankerBufLen)
+
+		engine.rankerRankChans[shard] = make(
+			chan rankerRankReq, options.RankerBufLen)
+
+		engine.rankerRemoveDocChans[shard] = make(
+			chan rankerRemoveDocReq, options.RankerBufLen)
+	}
+}
+
+// InitStore initialize the persistent store channel
+func (engine *Engine) InitStore() {
+	engine.storeIndexDocChans = make(
+		[]chan storeIndexDocReq, engine.initOptions.StoreShards)
+
+	for shard := 0; shard < engine.initOptions.StoreShards; shard++ {
+		engine.storeIndexDocChans[shard] = make(
+			chan storeIndexDocReq)
+	}
+	engine.storeInitChan = make(
+		chan bool, engine.initOptions.StoreShards)
+}
+
+// CheckMem check the memory when the memory is larger
+// than 99.99% using the store
+func (engine *Engine) CheckMem() {
+	// Todo test
+	if !engine.initOptions.UseStore {
+		log.Println("Check virtualMemory...")
+
+		vmem, _ := mem.VirtualMemory()
+		log.Printf("Total: %v, Free: %v, UsedPercent: %f%%\n",
+			vmem.Total, vmem.Free, vmem.UsedPercent)
+
+		useMem := fmt.Sprintf("%.2f", vmem.UsedPercent)
+		if useMem == "99.99" {
+			engine.initOptions.UseStore = true
+			engine.initOptions.StoreFolder = DefaultPath
+			// os.MkdirAll(DefaultPath, 0777)
+		}
+	}
+}
+
+// Store start the persistent store work connection
+func (engine *Engine) Store() {
+	// if engine.initOptions.UseStore {
+	err := os.MkdirAll(engine.initOptions.StoreFolder, 0700)
+	if err != nil {
+		log.Fatalf("Can not create directory: %s ; %v",
+			engine.initOptions.StoreFolder, err)
+	}
+
+	// 打开或者创建数据库
+	engine.dbs = make([]store.Store, engine.initOptions.StoreShards)
+	for shard := 0; shard < engine.initOptions.StoreShards; shard++ {
+		dbPath := engine.initOptions.StoreFolder + "/" +
+			StoreFilePrefix + "." + strconv.Itoa(shard)
+
+		db, err := store.OpenStore(dbPath, engine.initOptions.StoreEngine)
+		if db == nil || err != nil {
+			log.Fatal("Unable to open database ", dbPath, ": ", err)
+		}
+		engine.dbs[shard] = db
+	}
+
+	// 从数据库中恢复
+	for shard := 0; shard < engine.initOptions.StoreShards; shard++ {
+		go engine.storeInitWorker(shard)
+	}
+
+	// 等待恢复完成
+	for shard := 0; shard < engine.initOptions.StoreShards; shard++ {
+		<-engine.storeInitChan
+	}
+
+	for {
+		runtime.Gosched()
+
+		engine.loc.RLock()
+		numDoced := engine.numIndexingReqs == engine.numDocsIndexed
+		engine.loc.RUnlock()
+
+		if numDoced {
+			break
+		}
+
+	}
+
+	// 关闭并重新打开数据库
+	for shard := 0; shard < engine.initOptions.StoreShards; shard++ {
+		engine.dbs[shard].Close()
+		dbPath := engine.initOptions.StoreFolder + "/" +
+			StoreFilePrefix + "." + strconv.Itoa(shard)
+
+		db, err := store.OpenStore(dbPath, engine.initOptions.StoreEngine)
+		if db == nil || err != nil {
+			log.Fatal("Unable to open database ", dbPath, ": ", err)
+		}
+		engine.dbs[shard] = db
+	}
+
+	for shard := 0; shard < engine.initOptions.StoreShards; shard++ {
+		go engine.storeIndexDocWorker(shard)
+	}
+	// }
+}
+
+// WithGse Using user defined segmenter
+// If using a not nil segmenter and the dictionary is loaded,
+// the `opt.GseDict` will be ignore.
+func (engine *Engine) WithGse(segmenter gse.Segmenter) *Engine {
+	if engine.initialized {
+		log.Fatal(`Do not re-initialize the engine, 
+			WithGse should call before initialize the engine.`)
+	}
+
+	engine.segmenter = segmenter
+	engine.loaded = true
+	return engine
+}
+
+// Init initialize the engine
+func (engine *Engine) Init(options types.EngineOpts) {
+	// 将线程数设置为CPU数
+	// runtime.GOMAXPROCS(runtime.NumCPU())
+	// runtime.GOMAXPROCS(128)
+
+	// 初始化初始参数
+	if engine.initialized {
+		log.Fatal("Do not re-initialize the engine.")
+	}
+
+	if options.GseDict == "" && !options.NotUseGse && !engine.loaded {
+		log.Printf("Dictionary file path is empty, load the default dictionary file.")
+		options.GseDict = "zh"
+	}
+
+	if options.UseStore == true && options.StoreFolder == "" {
+		log.Printf("Store file path is empty, use default folder path.")
+		options.StoreFolder = DefaultPath
+		// os.MkdirAll(DefaultPath, 0777)
+	}
+
+	options.Init()
+	engine.initOptions = options
+	engine.initialized = true
+
+	if !options.NotUseGse {
+		if !engine.loaded {
+			// 载入分词器词典
+			engine.segmenter.LoadDict(options.GseDict)
+			engine.loaded = true
+		}
+
+		// 初始化停用词
+		engine.stopTokens.Init(options.StopTokenFile)
+	}
+
+	// 初始化索引器和排序器
+	for shard := 0; shard < options.NumShards; shard++ {
+		engine.indexers = append(engine.indexers, core.Indexer{})
+		engine.indexers[shard].Init(*options.IndexerOpts)
+
+		engine.rankers = append(engine.rankers, core.Ranker{})
+		engine.rankers[shard].Init(options.IDOnly)
+	}
+
+	// 初始化分词器通道
+	engine.segmenterChan = make(
+		chan segmenterReq, options.NumGseThreads)
+
+	// 初始化索引器通道
+	engine.Indexer(options)
+
+	// 初始化排序器通道
+	engine.Ranker(options)
+
+	// engine.CheckMem(engine.initOptions.UseStore)
+	engine.CheckMem()
+
+	// 初始化持久化存储通道
+	if engine.initOptions.UseStore {
+		engine.InitStore()
+	}
+
+	// 启动分词器
+	for iThread := 0; iThread < options.NumGseThreads; iThread++ {
+		go engine.segmenterWorker()
+	}
+
+	// 启动索引器和排序器
+	for shard := 0; shard < options.NumShards; shard++ {
+		go engine.indexerAddDocWorker(shard)
+		go engine.indexerRemoveDocWorker(shard)
+		go engine.rankerAddDocWorker(shard)
+		go engine.rankerRemoveDocWorker(shard)
+
+		for i := 0; i < options.NumIndexerThreadsPerShard; i++ {
+			go engine.indexerLookupWorker(shard)
+		}
+		for i := 0; i < options.NumRankerThreadsPerShard; i++ {
+			go engine.rankerRankWorker(shard)
+		}
+	}
+
+	// 启动持久化存储工作协程
+	if engine.initOptions.UseStore {
+		engine.Store()
+	}
+
+	atomic.AddUint64(&engine.numDocsStored, engine.numIndexingReqs)
+}
+
+// IndexDoc add the document to the index
+// 将文档加入索引
+//
+// 输入参数：
+//  docId	      标识文档编号，必须唯一，docId == 0 表示非法文档（用于强制刷新索引），[1, +oo) 表示合法文档
+//  data	      见 DocIndexData 注释
+//  forceUpdate 是否强制刷新 cache，如果设为 true，则尽快添加到索引，否则等待 cache 满之后一次全量添加
+//
+// 注意：
+//      1. 这个函数是线程安全的，请尽可能并发调用以提高索引速度
+//      2. 这个函数调用是非同步的，也就是说在函数返回时有可能文档还没有加入索引中，因此
+//         如果立刻调用Search可能无法查询到这个文档。强制刷新索引请调用FlushIndex函数。
+func (engine *Engine) IndexDoc(docId uint64, data types.DocData,
+	forceUpdate ...bool) {
+	engine.Index(docId, data, forceUpdate...)
+}
+
+// Index add the document to the index
+func (engine *Engine) Index(docId uint64, data types.DocData,
+	forceUpdate ...bool) {
+
+	var force bool
+	if len(forceUpdate) > 0 {
+		force = forceUpdate[0]
+	}
+
+	// if engine.HasDoc(docId) {
+	// 	engine.RemoveDoc(docId)
+	// }
+
+	// data.Tokens
+	engine.internalIndexDoc(docId, data, force)
+
+	hash := murmur.Sum32(fmt.Sprintf("%d", docId)) %
+		uint32(engine.initOptions.StoreShards)
+
+	if engine.initOptions.UseStore && docId != 0 {
+		engine.storeIndexDocChans[hash] <- storeIndexDocReq{
+			docId: docId, data: data}
+	}
+}
+
+func (engine *Engine) internalIndexDoc(docId uint64, data types.DocData,
+	forceUpdate bool) {
+
+	if !engine.initialized {
+		log.Fatal("The engine must be initialized first.")
+	}
+
+	if docId != 0 {
+		atomic.AddUint64(&engine.numIndexingReqs, 1)
+	}
+	if forceUpdate {
+		atomic.AddUint64(&engine.numForceUpdatingReqs, 1)
+	}
+
+	hash := murmur.Sum32(fmt.Sprintf("%d%s", docId, data.Content))
+	engine.segmenterChan <- segmenterReq{
+		docId: docId, hash: hash, data: data, forceUpdate: forceUpdate}
+}
+
+// RemoveDoc remove the document from the index
+// 将文档从索引中删除
+//
+// 输入参数：
+//  docId	      标识文档编号，必须唯一，docId == 0 表示非法文档（用于强制刷新索引），[1, +oo) 表示合法文档
+//  forceUpdate 是否强制刷新 cache，如果设为 true，则尽快删除索引，否则等待 cache 满之后一次全量删除
+//
+// 注意：
+//      1. 这个函数是线程安全的，请尽可能并发调用以提高索引速度
+//      2. 这个函数调用是非同步的，也就是说在函数返回时有可能文档还没有加入索引中，因此
+//         如果立刻调用 Search 可能无法查询到这个文档。强制刷新索引请调用 FlushIndex 函数。
+func (engine *Engine) RemoveDoc(docId uint64, forceUpdate ...bool) {
+	var force bool
+	if len(forceUpdate) > 0 {
+		force = forceUpdate[0]
+	}
+
+	if !engine.initialized {
+		log.Fatal("The engine must be initialized first.")
+	}
+
+	if docId != 0 {
+		atomic.AddUint64(&engine.numRemovingReqs, 1)
+	}
+	if force {
+		atomic.AddUint64(&engine.numForceUpdatingReqs, 1)
+	}
+	for shard := 0; shard < engine.initOptions.NumShards; shard++ {
+		engine.indexerRemoveDocChans[shard] <- indexerRemoveDocReq{
+			docId: docId, forceUpdate: force}
+
+		if docId == 0 {
+			continue
+		}
+		engine.rankerRemoveDocChans[shard] <- rankerRemoveDocReq{docId: docId}
+	}
+
+	if engine.initOptions.UseStore && docId != 0 {
+		// 从数据库中删除
+		hash := murmur.Sum32(fmt.Sprintf("%d", docId)) %
+			uint32(engine.initOptions.StoreShards)
+
+		go engine.storeRemoveDocWorker(docId, hash)
+	}
+}
+
+// // 获取文本的分词结果
+// func (engine *Engine) Tokens(text []byte) (tokens []string) {
+// 	querySegments := engine.segmenter.Segment(text)
+// 	for _, s := range querySegments {
+// 		token := s.Token().Text()
+// 		if !engine.stopTokens.IsStopToken(token) {
+// 			tokens = append(tokens, token)
+// 		}
+// 	}
+// 	return tokens
+// }
+
+// Segment get the word segmentation result of the text
+// 获取文本的分词结果, 只分词与过滤弃用词
+func (engine *Engine) Segment(content string) (keywords []string) {
+	segments := engine.segmenter.ModeSegment([]byte(content),
+		engine.initOptions.GseMode)
+
+	for _, segment := range segments {
+		token := segment.Token().Text()
+		if !engine.stopTokens.IsStopToken(token) {
+			keywords = append(keywords, token)
+		}
+	}
+
+	return
+}
+
+// Tokens get the engine tokens
+func (engine *Engine) Tokens(request types.SearchReq) (tokens []string) {
+	// 收集关键词
+	// tokens := []string{}
+	if request.Text != "" {
+		request.Text = strings.ToLower(request.Text)
+		if engine.initOptions.NotUseGse {
+			tokens = strings.Split(request.Text, " ")
+		} else {
+			// querySegments := engine.segmenter.Segment([]byte(request.Text))
+			// tokens = engine.Tokens([]byte(request.Text))
+			tokens = engine.Segment(request.Text)
+		}
+
+		// 叠加 tokens
+		for _, t := range request.Tokens {
+			tokens = append(tokens, t)
+		}
+
+		return
+	}
+
+	for _, t := range request.Tokens {
+		tokens = append(tokens, t)
+	}
+	return
+}
+
+// RankId rank docs by types.ScoredIDs
+func (engine *Engine) RankId(request types.SearchReq, RankOpts types.RankOpts,
+	tokens []string, rankerReturnChan chan rankerReturnReq) (
+	output types.SearchResp) {
+	// 从通信通道读取排序器的输出
+	numDocs := 0
+	var rankOutput types.ScoredIDs
+	// var rankOutput interface{}
+
+	//**********/ begin
+	timeout := request.Timeout
+	isTimeout := false
+	if timeout <= 0 {
+		// 不设置超时
+		for shard := 0; shard < engine.initOptions.NumShards; shard++ {
+			rankerOutput := <-rankerReturnChan
+			if !request.CountDocsOnly {
+				if rankerOutput.docs != nil {
+					for _, doc := range rankerOutput.docs.(types.ScoredIDs) {
+						rankOutput = append(rankOutput, doc)
+					}
+				}
+			}
+			numDocs += rankerOutput.numDocs
+		}
+	} else {
+		// 设置超时
+		deadline := time.Now().Add(time.Nanosecond *
+			time.Duration(NumNanosecondsInAMillisecond*request.Timeout))
+
+		for shard := 0; shard < engine.initOptions.NumShards; shard++ {
+			select {
+			case rankerOutput := <-rankerReturnChan:
+				if !request.CountDocsOnly {
+					if rankerOutput.docs != nil {
+						for _, doc := range rankerOutput.docs.(types.ScoredIDs) {
+							rankOutput = append(rankOutput, doc)
+						}
+					}
+				}
+				numDocs += rankerOutput.numDocs
+			case <-time.After(deadline.Sub(time.Now())):
+				isTimeout = true
+				break
+			}
+		}
+	}
+
+	// 再排序
+	if !request.CountDocsOnly && !request.Orderless {
+		if RankOpts.ReverseOrder {
+			sort.Sort(sort.Reverse(rankOutput))
+		} else {
+			sort.Sort(rankOutput)
+		}
+	}
+
+	// 准备输出
+	output.Tokens = tokens
+	// 仅当 CountDocsOnly 为 false 时才充填 output.Docs
+	if !request.CountDocsOnly {
+		if request.Orderless {
+			// 无序状态无需对 Offset 截断
+			output.Docs = rankOutput
+		} else {
+			var start, end int
+			if RankOpts.MaxOutputs == 0 {
+				start = utils.MinInt(RankOpts.OutputOffset, len(rankOutput))
+				end = len(rankOutput)
+			} else {
+				start = utils.MinInt(RankOpts.OutputOffset, len(rankOutput))
+				end = utils.MinInt(start+RankOpts.MaxOutputs, len(rankOutput))
+			}
+			output.Docs = rankOutput[start:end]
+		}
+	}
+
+	output.NumDocs = numDocs
+	output.Timeout = isTimeout
+
+	return
+}
+
+// Ranks rank docs by types.ScoredDocs
+func (engine *Engine) Ranks(request types.SearchReq, RankOpts types.RankOpts,
+	tokens []string, rankerReturnChan chan rankerReturnReq) (
+	output types.SearchResp) {
+	// 从通信通道读取排序器的输出
+	numDocs := 0
+	rankOutput := types.ScoredDocs{}
+
+	//**********/ begin
+	timeout := request.Timeout
+	isTimeout := false
+	if timeout <= 0 {
+		// 不设置超时
+		for shard := 0; shard < engine.initOptions.NumShards; shard++ {
+			rankerOutput := <-rankerReturnChan
+			if !request.CountDocsOnly {
+				if rankerOutput.docs != nil {
+					for _, doc := range rankerOutput.docs.(types.ScoredDocs) {
+						rankOutput = append(rankOutput, doc)
+					}
+				}
+			}
+			numDocs += rankerOutput.numDocs
+		}
+	} else {
+		// 设置超时
+		deadline := time.Now().Add(time.Nanosecond *
+			time.Duration(NumNanosecondsInAMillisecond*request.Timeout))
+
+		for shard := 0; shard < engine.initOptions.NumShards; shard++ {
+			select {
+			case rankerOutput := <-rankerReturnChan:
+				if !request.CountDocsOnly {
+					if rankerOutput.docs != nil {
+						for _, doc := range rankerOutput.docs.(types.ScoredDocs) {
+							rankOutput = append(rankOutput, doc)
+						}
+					}
+				}
+				numDocs += rankerOutput.numDocs
+			case <-time.After(deadline.Sub(time.Now())):
+				isTimeout = true
+				break
+			}
+		}
+	}
+
+	// 再排序
+	if !request.CountDocsOnly && !request.Orderless {
+		if RankOpts.ReverseOrder {
+			sort.Sort(sort.Reverse(rankOutput))
+		} else {
+			sort.Sort(rankOutput)
+		}
+	}
+
+	// 准备输出
+	output.Tokens = tokens
+	// 仅当 CountDocsOnly 为 false 时才充填 output.Docs
+	if !request.CountDocsOnly {
+		if request.Orderless {
+			// 无序状态无需对 Offset 截断
+			output.Docs = rankOutput
+		} else {
+			var start, end int
+			if RankOpts.MaxOutputs == 0 {
+				start = utils.MinInt(RankOpts.OutputOffset, len(rankOutput))
+				end = len(rankOutput)
+			} else {
+				start = utils.MinInt(RankOpts.OutputOffset, len(rankOutput))
+				end = utils.MinInt(start+RankOpts.MaxOutputs, len(rankOutput))
+			}
+			output.Docs = rankOutput[start:end]
+		}
+	}
+
+	output.NumDocs = numDocs
+	output.Timeout = isTimeout
+
+	return
+}
+
+// Search find the document that satisfies the search criteria.
+// This function is thread safe
+// 查找满足搜索条件的文档，此函数线程安全
+func (engine *Engine) Search(request types.SearchReq) (output types.SearchResp) {
+	if !engine.initialized {
+		log.Fatal("The engine must be initialized first.")
+	}
+
+	tokens := engine.Tokens(request)
+
+	var RankOpts types.RankOpts
+	if request.RankOpts == nil {
+		RankOpts = *engine.initOptions.DefaultRankOpts
+	} else {
+		RankOpts = *request.RankOpts
+	}
+	if RankOpts.ScoringCriteria == nil {
+		RankOpts.ScoringCriteria = engine.initOptions.DefaultRankOpts.ScoringCriteria
+	}
+
+	// 建立排序器返回的通信通道
+	rankerReturnChan := make(
+		chan rankerReturnReq, engine.initOptions.NumShards)
+
+	// 生成查找请求
+	lookupRequest := indexerLookupReq{
+		countDocsOnly:    request.CountDocsOnly,
+		tokens:           tokens,
+		labels:           request.Labels,
+		docIds:           request.DocIds,
+		options:          RankOpts,
+		rankerReturnChan: rankerReturnChan,
+		orderless:        request.Orderless,
+		logic:            request.Logic,
+	}
+
+	// 向索引器发送查找请求
+	for shard := 0; shard < engine.initOptions.NumShards; shard++ {
+		engine.indexerLookupChans[shard] <- lookupRequest
+	}
+
+	if engine.initOptions.IDOnly {
+		output = engine.RankId(request, RankOpts, tokens, rankerReturnChan)
+		return
+	}
+
+	output = engine.Ranks(request, RankOpts, tokens, rankerReturnChan)
+	return
+}
+
+// Flush block wait until all indexes are added
+// 阻塞等待直到所有索引添加完毕
+func (engine *Engine) Flush() {
+	for {
+		runtime.Gosched()
+
+		engine.loc.RLock()
+		inxd := engine.numIndexingReqs == engine.numDocsIndexed
+		rmd := engine.numRemovingReqs*uint64(engine.initOptions.NumShards) ==
+			engine.numDocsRemoved
+		stored := !engine.initOptions.UseStore || engine.numIndexingReqs ==
+			engine.numDocsStored
+		engine.loc.RUnlock()
+
+		if inxd && rmd && stored {
+			// 保证 CHANNEL 中 REQUESTS 全部被执行完
+			break
+		}
+	}
+
+	// 强制更新，保证其为最后的请求
+	engine.IndexDoc(0, types.DocData{}, true)
+	for {
+		runtime.Gosched()
+
+		engine.loc.RLock()
+		forced := engine.numForceUpdatingReqs*uint64(engine.initOptions.NumShards) ==
+			engine.numDocsForceUpdated
+		engine.loc.RUnlock()
+
+		if forced {
+			return
+		}
+
+	}
+}
+
+// FlushIndex block wait until all indexes are added
+// 阻塞等待直到所有索引添加完毕
+func (engine *Engine) FlushIndex() {
+	engine.Flush()
+}
+
+// Close close the engine
+// 关闭引擎
+func (engine *Engine) Close() {
+	engine.Flush()
+	if engine.initOptions.UseStore {
+		for _, db := range engine.dbs {
+			db.Close()
+		}
+	}
+}
+
+// 从文本hash得到要分配到的 shard
+func (engine *Engine) getShard(hash uint32) int {
+	return int(hash - hash/uint32(engine.initOptions.NumShards)*
+		uint32(engine.initOptions.NumShards))
+}

vendor/github.com/go-ego/riot/indexer_worker.go

@@ -0,0 +1,166 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package riot
+
+import (
+	"sync/atomic"
+
+	"github.com/go-ego/riot/types"
+)
+
+type indexerAddDocReq struct {
+	doc         *types.DocIndex
+	forceUpdate bool
+}
+
+type indexerLookupReq struct {
+	countDocsOnly    bool
+	tokens           []string
+	labels           []string
+	docIds           map[uint64]bool
+	options          types.RankOpts
+	rankerReturnChan chan rankerReturnReq
+	orderless        bool
+	logic            types.Logic
+}
+
+type indexerRemoveDocReq struct {
+	docId       uint64
+	forceUpdate bool
+}
+
+func (engine *Engine) indexerAddDocWorker(shard int) {
+	for {
+		request := <-engine.indexerAddDocChans[shard]
+		engine.indexers[shard].AddDocToCache(request.doc, request.forceUpdate)
+		if request.doc != nil {
+			atomic.AddUint64(&engine.numTokenIndexAdded,
+				uint64(len(request.doc.Keywords)))
+
+			engine.loc.Lock()
+			atomic.AddUint64(&engine.numDocsIndexed, 1)
+			// engine.numDocsIndexed++
+			engine.loc.Unlock()
+		}
+		if request.forceUpdate {
+			engine.loc.Lock()
+			atomic.AddUint64(&engine.numDocsForceUpdated, 1)
+			engine.loc.Unlock()
+		}
+	}
+}
+
+func (engine *Engine) indexerRemoveDocWorker(shard int) {
+	for {
+		request := <-engine.indexerRemoveDocChans[shard]
+		engine.indexers[shard].RemoveDocToCache(request.docId, request.forceUpdate)
+		if request.docId != 0 {
+			engine.loc.Lock()
+			atomic.AddUint64(&engine.numDocsRemoved, 1)
+			engine.loc.Unlock()
+		}
+		if request.forceUpdate {
+			engine.loc.Lock()
+			atomic.AddUint64(&engine.numDocsForceUpdated, 1)
+			engine.loc.Unlock()
+		}
+	}
+}
+
+func (engine *Engine) orderLess(
+	request indexerLookupReq, docs []types.IndexedDoc) {
+
+	if engine.initOptions.IDOnly {
+		var outputDocs []types.ScoredID
+		// var outputDocs types.ScoredIDs
+		for _, d := range docs {
+			outputDocs = append(outputDocs, types.ScoredID{
+				DocId:            d.DocId,
+				TokenSnippetLocs: d.TokenSnippetLocs,
+				TokenLocs:        d.TokenLocs})
+		}
+
+		request.rankerReturnChan <- rankerReturnReq{
+			docs:    types.ScoredIDs(outputDocs),
+			numDocs: len(outputDocs),
+		}
+
+		return
+	}
+
+	var outputDocs []types.ScoredDoc
+	// var outputDocs types.ScoredDocs
+	for _, d := range docs {
+		outputDocs = append(outputDocs, types.ScoredDoc{
+			DocId:            d.DocId,
+			TokenSnippetLocs: d.TokenSnippetLocs,
+			TokenLocs:        d.TokenLocs})
+	}
+
+	request.rankerReturnChan <- rankerReturnReq{
+		docs:    types.ScoredDocs(outputDocs),
+		numDocs: len(outputDocs),
+	}
+}
+
+func (engine *Engine) indexerLookupWorker(shard int) {
+	for {
+		request := <-engine.indexerLookupChans[shard]
+
+		var (
+			docs    []types.IndexedDoc
+			numDocs int
+		)
+		if request.docIds == nil {
+			docs, numDocs = engine.indexers[shard].Lookup(
+				request.tokens, request.labels,
+				nil, request.countDocsOnly, request.logic)
+			// docs, numDocs = engine.indexers[shard].Lookup(request.tokens,
+			// request.labels, nil, request.countDocsOnly)
+		} else {
+			docs, numDocs = engine.indexers[shard].Lookup(
+				request.tokens, request.labels,
+				request.docIds, request.countDocsOnly, request.logic)
+			// docs, numDocs = engine.indexers[shard].Lookup(request.tokens,
+			// request.labels, request.docIds, request.countDocsOnly)
+		}
+
+		if request.countDocsOnly {
+			request.rankerReturnChan <- rankerReturnReq{numDocs: numDocs}
+			continue
+		}
+
+		if len(docs) == 0 {
+			request.rankerReturnChan <- rankerReturnReq{}
+			continue
+		}
+
+		if request.orderless {
+			// var outputDocs interface{}
+			engine.orderLess(request, docs)
+
+			continue
+		}
+
+		rankerRequest := rankerRankReq{
+			countDocsOnly:    request.countDocsOnly,
+			docs:             docs,
+			options:          request.options,
+			rankerReturnChan: request.rankerReturnChan,
+		}
+		engine.rankerRankChans[shard] <- rankerRequest
+	}
+}

vendor/github.com/go-ego/riot/info.go

@@ -0,0 +1,149 @@
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package riot
+
+import (
+	"sync"
+
+	"github.com/go-vgo/gt/info"
+)
+
+var (
+	lck sync.RWMutex
+
+	// InitMemUsed init mem used
+	InitMemUsed uint64
+	// InitDiskUsed init disk used
+	InitDiskUsed uint64
+)
+
+func init() {
+	lck.Lock()
+	InitMemUsed, _ = MemUsed()
+	InitDiskUsed, _ = DiskUsed()
+	lck.Unlock()
+}
+
+// MemPercent returns the amount of use memory in percent.
+func MemPercent() (string, error) {
+	return info.MemPercent()
+}
+
+// MemUsed returns the amount of used memory in bytes.
+func MemUsed() (uint64, error) {
+	return info.MemUsed()
+}
+
+// UsedMem returns the amount of riot used memory in bytes
+// after init() func.
+func (engine *Engine) UsedMem() (uint64, error) {
+	memUsed, err := MemUsed()
+	if err != nil {
+		return 0, err
+	}
+
+	return memUsed - InitMemUsed, err
+}
+
+// MemTotal returns the amount of total memory in bytes.
+func MemTotal() (uint64, error) {
+	return info.MemTotal()
+}
+
+// MemFree returns the amount of free memory in bytes.
+func MemFree() (uint64, error) {
+	return info.MemFree()
+}
+
+// ToKB bytes to kb
+func ToKB(data uint64) uint64 {
+	return data / 1024
+}
+
+// ToMB bytes to mb
+func ToMB(data uint64) uint64 {
+	return data / 1024 / 1024
+}
+
+// ToGB bytes to gb
+func ToGB(data uint64) uint64 {
+	return data / 1024 / 1024 / 1024
+}
+
+// Disk init the disk
+// func Disk(pt ...bool) ([]*disk.UsageStat, error) {
+// 	return info.Disk(pt...)
+// }
+
+// DiskPercent returns the amount of use disk in percent.
+func DiskPercent() (string, error) {
+	return info.DiskPercent()
+}
+
+// DiskUsed returns the amount of use disk in bytes.
+func DiskUsed() (uint64, error) {
+	return info.DiskUsed()
+}
+
+// UsedDisk returns the amount of use disk in bytes
+// after init() func.
+func (engine *Engine) UsedDisk() (uint64, error) {
+	diskUsed, err := DiskUsed()
+	if err != nil {
+		return 0, err
+	}
+
+	return diskUsed - InitDiskUsed, err
+}
+
+// DiskTotal returns the amount of total disk in bytes.
+func DiskTotal() (uint64, error) {
+	return info.DiskTotal()
+}
+
+// DiskFree returns the amount of free disk in bytes.
+func DiskFree() (uint64, error) {
+	return info.DiskFree()
+}
+
+// CPUInfo returns the cpu info
+func CPUInfo(args ...int) (string, error) {
+	return info.CPUInfo(args...)
+}
+
+// CPUPercent returns the amount of use cpu in percent.
+func CPUPercent() ([]float64, error) {
+	return info.CPUPercent()
+}
+
+// Uptime returns the system uptime in seconds.
+func Uptime() (uptime uint64, err error) {
+	return info.Uptime()
+}
+
+// PlatformInfo fetches system platform information.
+func PlatformInfo() (platform, family, osVersion string, err error) {
+	return info.PlatformInfo()
+}
+
+// Platform returns the platform name and OS Version.
+func Platform() (string, error) {
+	return info.Platform()
+}
+
+// KernelVer returns the kernel version as a string.
+func KernelVer() (string, error) {
+	return info.KernelVer()
+}

vendor/github.com/go-ego/riot/ranker_worker.go

@@ -0,0 +1,82 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package riot
+
+import (
+	"github.com/go-ego/riot/types"
+)
+
+type rankerAddDocReq struct {
+	docId  uint64
+	fields interface{}
+	// new
+	content string
+	// new 属性
+	attri interface{}
+}
+
+type rankerRankReq struct {
+	docs             []types.IndexedDoc
+	options          types.RankOpts
+	rankerReturnChan chan rankerReturnReq
+	countDocsOnly    bool
+}
+
+type rankerReturnReq struct {
+	// docs    types.ScoredDocs
+	docs    interface{}
+	numDocs int
+}
+
+type rankerRemoveDocReq struct {
+	docId uint64
+}
+
+func (engine *Engine) rankerAddDocWorker(shard int) {
+	for {
+		request := <-engine.rankerAddDocChans[shard]
+		if engine.initOptions.IDOnly {
+			engine.rankers[shard].AddDoc(request.docId, request.fields)
+			return
+		}
+		// } else {
+		engine.rankers[shard].AddDoc(request.docId, request.fields,
+			request.content, request.attri)
+		// }
+	}
+}
+
+func (engine *Engine) rankerRankWorker(shard int) {
+	for {
+		request := <-engine.rankerRankChans[shard]
+		if request.options.MaxOutputs != 0 {
+			request.options.MaxOutputs += request.options.OutputOffset
+		}
+		request.options.OutputOffset = 0
+		outputDocs, numDocs := engine.rankers[shard].Rank(request.docs,
+			request.options, request.countDocsOnly)
+
+		request.rankerReturnChan <- rankerReturnReq{
+			docs: outputDocs, numDocs: numDocs}
+	}
+}
+
+func (engine *Engine) rankerRemoveDocWorker(shard int) {
+	for {
+		request := <-engine.rankerRemoveDocChans[shard]
+		engine.rankers[shard].RemoveDoc(request.docId)
+	}
+}

vendor/github.com/go-ego/riot/riot.go

@@ -0,0 +1,198 @@
+// Copyright 2017 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package riot
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"encoding/binary"
+	"encoding/gob"
+
+	"github.com/go-ego/murmur"
+	"github.com/go-ego/riot/core"
+	"github.com/go-ego/riot/types"
+	toml "github.com/go-vgo/gt/conf"
+)
+
+// New create a new engine with mode
+func New(conf ...interface{}) *Engine {
+	// func (engine *Engine) New(conf com.Config) *Engine{
+	if len(conf) > 0 && strings.HasSuffix(conf[0].(string), ".toml") {
+		var (
+			config   types.EngineOpts
+			searcher = &Engine{}
+		)
+
+		fs := conf[0].(string)
+		log.Println("conf path is: ", fs)
+		toml.Init(fs, &config)
+		go toml.Watch(fs, &config)
+
+		searcher.Init(config)
+		return searcher
+	}
+
+	return NewEngine(conf...)
+}
+
+// NewEngine create a new engine
+func NewEngine(conf ...interface{}) *Engine {
+	var (
+		searcher = &Engine{}
+
+		path          = DefaultPath
+		storageShards = 10
+		numShards     = 10
+
+		segmentDict string
+	)
+
+	if len(conf) > 0 {
+		segmentDict = conf[0].(string)
+	}
+
+	if len(conf) > 1 {
+		path = conf[1].(string)
+	}
+
+	if len(conf) > 2 {
+		numShards = conf[2].(int)
+		storageShards = conf[2].(int)
+	}
+
+	searcher.Init(types.EngineOpts{
+		// Using:         using,
+		StoreShards: storageShards,
+		NumShards:   numShards,
+		IndexerOpts: &types.IndexerOpts{
+			IndexType: types.DocIdsIndex,
+		},
+		UseStore:    true,
+		StoreFolder: path,
+		// StoreEngine: storageEngine,
+		GseDict: segmentDict,
+		// StopTokenFile: stopTokenFile,
+	})
+
+	// defer searcher.Close()
+	os.MkdirAll(path, 0777)
+
+	// 等待索引刷新完毕
+	// searcher.Flush()
+	// log.Println("recover index number: ", searcher.NumDocsIndexed())
+
+	return searcher
+}
+
+// func (engine *Engine) IsDocExist(docId uint64) bool {
+// 	return core.IsDocExist(docId)
+// }
+
+// HasDoc if the document is exist return true
+func (engine *Engine) HasDoc(docId uint64) bool {
+	for shard := 0; shard < engine.initOptions.NumShards; shard++ {
+		engine.indexers = append(engine.indexers, core.Indexer{})
+
+		has := engine.indexers[shard].HasDoc(docId)
+
+		if has {
+			return true
+		}
+	}
+
+	return false
+}
+
+// HasDocDB if the document is exist in the database
+// return true
+func (engine *Engine) HasDocDB(docId uint64) bool {
+	b := make([]byte, 10)
+	length := binary.PutUvarint(b, docId)
+
+	shard := murmur.Sum32(fmt.Sprintf("%d", docId)) %
+		uint32(engine.initOptions.StoreShards)
+
+	has, err := engine.dbs[shard].Has(b[0:length])
+	if err != nil {
+		log.Println("engine.dbs[shard].Has(b[0:length]): ", err)
+	}
+
+	return has
+}
+
+// GetDBAllIds get all the DocId from the storage database
+// and return
+// 从数据库遍历所有的 DocId, 并返回
+func (engine *Engine) GetDBAllIds() []uint64 {
+	docsId := make([]uint64, 0)
+	for i := range engine.dbs {
+		engine.dbs[i].ForEach(func(k, v []byte) error {
+			// fmt.Println(k, v)
+			docId, _ := binary.Uvarint(k)
+			docsId = append(docsId, docId)
+			return nil
+		})
+	}
+
+	return docsId
+}
+
+// GetDBAllDocs get the db all docs
+func (engine *Engine) GetDBAllDocs() (
+	docsId []uint64, docsData []types.DocData) {
+	for i := range engine.dbs {
+		engine.dbs[i].ForEach(func(key, val []byte) error {
+			// fmt.Println(k, v)
+			docId, _ := binary.Uvarint(key)
+			docsId = append(docsId, docId)
+
+			buf := bytes.NewReader(val)
+			dec := gob.NewDecoder(buf)
+
+			var data types.DocData
+			err := dec.Decode(&data)
+			if err != nil {
+				log.Println("dec.decode: ", err)
+			}
+
+			docsData = append(docsData, data)
+
+			return nil
+		})
+	}
+
+	return docsId, docsData
+}
+
+// GetAllDocIds get all the DocId from the storage database
+// and return
+// 从数据库遍历所有的 DocId, 并返回
+func (engine *Engine) GetAllDocIds() []uint64 {
+	return engine.GetDBAllIds()
+}
+
+// Try handler(err)
+func Try(fun func(), handler func(interface{})) {
+	defer func() {
+		if err := recover(); err != nil {
+			handler(err)
+		}
+	}()
+	fun()
+}

vendor/github.com/go-ego/riot/riot_pkg.go

@@ -0,0 +1,30 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+/*
+
+Package riot full text search engine
+*/
+package riot
+
+import (
+	// _ "github.com/cznic/kv"
+	_ "github.com/coreos/bbolt"
+	// _ "github.com/boltdb/bolt"
+	_ "github.com/dgraph-io/badger"
+	_ "github.com/go-ego/gse"
+	_ "github.com/go-ego/murmur"
+	_ "github.com/syndtr/goleveldb/leveldb"
+)

vendor/github.com/go-ego/riot/segment.go

@@ -0,0 +1,349 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package riot
+
+import (
+	// "fmt"
+
+	"strings"
+
+	"github.com/go-ego/gpy"
+	"github.com/go-ego/riot/types"
+)
+
+// TMap defines the tokens map type map[string][]int
+type TMap map[string][]int
+
+type segmenterReq struct {
+	docId uint64
+	hash  uint32
+	data  types.DocData
+	// data        types.DocumentIndexData
+	forceUpdate bool
+}
+
+// ForSplitData for split segment's data, segspl
+func (engine *Engine) ForSplitData(strData []string, num int) (TMap, int) {
+	var (
+		numTokens int
+		splitStr  string
+	)
+	tokensMap := make(map[string][]int)
+
+	for i := 0; i < num; i++ {
+		if strData[i] != "" {
+			if !engine.stopTokens.IsStopToken(strData[i]) {
+				numTokens++
+				tokensMap[strData[i]] = append(tokensMap[strData[i]], numTokens)
+			}
+
+			splitStr += strData[i]
+			if !engine.stopTokens.IsStopToken(splitStr) {
+				numTokens++
+				tokensMap[splitStr] = append(tokensMap[splitStr], numTokens)
+			}
+
+			if engine.initOptions.Using == 6 {
+				// more combination
+				var splitsStr string
+				for s := i + 1; s < len(strData); s++ {
+					splitsStr += strData[s]
+
+					if !engine.stopTokens.IsStopToken(splitsStr) {
+						numTokens++
+						tokensMap[splitsStr] = append(tokensMap[splitsStr], numTokens)
+					}
+				}
+			}
+
+		}
+	}
+
+	return tokensMap, numTokens
+}
+
+func (engine *Engine) splitData(request segmenterReq) (TMap, int) {
+	var (
+		num       int
+		numTokens int
+	)
+	tokensMap := make(map[string][]int)
+
+	if request.data.Content != "" {
+		content := strings.ToLower(request.data.Content)
+		if engine.initOptions.Using == 3 {
+			// use segmenter
+			segments := engine.segmenter.ModeSegment([]byte(content),
+				engine.initOptions.GseMode)
+
+			for _, segment := range segments {
+				token := segment.Token().Text()
+				if !engine.stopTokens.IsStopToken(token) {
+					tokensMap[token] = append(tokensMap[token], segment.Start())
+				}
+			}
+			numTokens += len(segments)
+		}
+
+		if engine.initOptions.Using == 4 {
+			tokensMap, numTokens = engine.defaultTokens(content)
+		}
+
+		if engine.initOptions.Using != 4 {
+			strData := strings.Split(content, "")
+			num = len(strData)
+			tokenMap, numToken := engine.ForSplitData(strData, num)
+			numTokens += numToken
+			for key, val := range tokenMap {
+				tokensMap[key] = val
+			}
+		}
+	}
+
+	for _, t := range request.data.Tokens {
+		if !engine.stopTokens.IsStopToken(t.Text) {
+			tokensMap[t.Text] = t.Locations
+		}
+	}
+
+	numTokens += len(request.data.Tokens)
+
+	return tokensMap, numTokens
+}
+
+func (engine *Engine) segmenterData(request segmenterReq) (TMap, int) {
+	tokensMap := make(map[string][]int)
+	numTokens := 0
+
+	if engine.initOptions.Using == 0 && request.data.Content != "" {
+		// Content 分词, 当文档正文不为空时，优先从内容分词中得到关键词
+		segments := engine.segmenter.ModeSegment([]byte(request.data.Content),
+			engine.initOptions.GseMode)
+
+		for _, segment := range segments {
+			token := segment.Token().Text()
+			if !engine.stopTokens.IsStopToken(token) {
+				tokensMap[token] = append(tokensMap[token], segment.Start())
+			}
+		}
+
+		for _, t := range request.data.Tokens {
+			if !engine.stopTokens.IsStopToken(t.Text) {
+				tokensMap[t.Text] = t.Locations
+			}
+		}
+
+		numTokens = len(segments) + len(request.data.Tokens)
+
+		return tokensMap, numTokens
+	}
+
+	if engine.initOptions.Using == 1 && request.data.Content != "" {
+		// Content 分词, 当文档正文不为空时，优先从内容分词中得到关键词
+		segments := engine.segmenter.ModeSegment([]byte(request.data.Content),
+			engine.initOptions.GseMode)
+
+		for _, segment := range segments {
+			token := segment.Token().Text()
+			if !engine.stopTokens.IsStopToken(token) {
+				tokensMap[token] = append(tokensMap[token], segment.Start())
+			}
+		}
+		numTokens = len(segments)
+
+		return tokensMap, numTokens
+	}
+
+	if engine.initOptions.Using == 2 ||
+		((engine.initOptions.Using == 1 || engine.initOptions.Using == 3) &&
+			request.data.Content == "") {
+		for _, t := range request.data.Tokens {
+			if !engine.stopTokens.IsStopToken(t.Text) {
+				tokensMap[t.Text] = t.Locations
+			}
+		}
+
+		numTokens = len(request.data.Tokens)
+
+		return tokensMap, numTokens
+	}
+
+	tokenMap, lenSplitData := engine.splitData(request)
+
+	return tokenMap, lenSplitData
+}
+
+func (engine *Engine) defaultTokens(content string) (tokensMap TMap, numTokens int) {
+	// use segmenter
+	tokensMap = make(map[string][]int)
+	strData := strings.Split(content, " ")
+	num := len(strData)
+	// if num == 1 {
+	// 	tokensMap[request.data.Content] = []int{1}
+	// }
+
+	if num > 0 {
+		tokenMap, numToken := engine.ForSplitData(strData, num)
+		numTokens += numToken
+
+		for key, val := range tokenMap {
+			tokensMap[key] = val
+		}
+	}
+
+	return
+}
+
+func (engine *Engine) segmenterWorker() {
+	for {
+		request := <-engine.segmenterChan
+		if request.docId == 0 {
+			if request.forceUpdate {
+				for i := 0; i < engine.initOptions.NumShards; i++ {
+					engine.indexerAddDocChans[i] <- indexerAddDocReq{
+						forceUpdate: true}
+				}
+			}
+			continue
+		}
+
+		shard := engine.getShard(request.hash)
+		tokensMap := make(map[string][]int)
+		numTokens := 0
+		if !(engine.initOptions.NotUseGse && engine.initOptions.Using == 0) {
+			tokensMap, numTokens = engine.segmenterData(request)
+		} else {
+			if request.data.Content != "" {
+				content := strings.ToLower(request.data.Content)
+				tokensMap, numTokens = engine.defaultTokens(content)
+			}
+
+			for _, t := range request.data.Tokens {
+				if !engine.stopTokens.IsStopToken(t.Text) {
+					tokensMap[t.Text] = t.Locations
+				}
+			}
+
+			numTokens += len(request.data.Tokens)
+		}
+
+		// 加入非分词的文档标签
+		for _, label := range request.data.Labels {
+			if !engine.initOptions.NotUseGse {
+				if !engine.stopTokens.IsStopToken(label) {
+					// 当正文中已存在关键字时，若不判断，位置信息将会丢失
+					if _, ok := tokensMap[label]; !ok {
+						tokensMap[label] = []int{}
+					}
+				}
+			} else {
+				// 当正文中已存在关键字时，若不判断，位置信息将会丢失
+				if _, ok := tokensMap[label]; !ok {
+					tokensMap[label] = []int{}
+				}
+			}
+		}
+
+		indexerRequest := indexerAddDocReq{
+			doc: &types.DocIndex{
+				DocId:    request.docId,
+				TokenLen: float32(numTokens),
+				Keywords: make([]types.KeywordIndex, len(tokensMap)),
+			},
+			forceUpdate: request.forceUpdate,
+		}
+		iTokens := 0
+		for k, v := range tokensMap {
+			indexerRequest.doc.Keywords[iTokens] = types.KeywordIndex{
+				Text: k,
+				// 非分词标注的词频设置为0，不参与tf-idf计算
+				Frequency: float32(len(v)),
+				Starts:    v}
+			iTokens++
+		}
+
+		engine.indexerAddDocChans[shard] <- indexerRequest
+		if request.forceUpdate {
+			for i := 0; i < engine.initOptions.NumShards; i++ {
+				if i == shard {
+					continue
+				}
+				engine.indexerAddDocChans[i] <- indexerAddDocReq{forceUpdate: true}
+			}
+		}
+		rankerRequest := rankerAddDocReq{
+			// docId: request.docId, fields: request.data.Fields}
+			docId: request.docId, fields: request.data.Fields,
+			content: request.data.Content, attri: request.data.Attri}
+		engine.rankerAddDocChans[shard] <- rankerRequest
+	}
+}
+
+// PinYin get the Chinese alphabet and abbreviation
+func (engine *Engine) PinYin(hans string) []string {
+	var (
+		str      string
+		pyStr    string
+		strArr   []string
+		splitStr string
+		// splitArr []string
+	)
+
+	//
+	splitHans := strings.Split(hans, "")
+	for i := 0; i < len(splitHans); i++ {
+		if splitHans[i] != "" {
+			if !engine.stopTokens.IsStopToken(splitHans[i]) {
+				strArr = append(strArr, splitHans[i])
+			}
+			splitStr += splitHans[i]
+		}
+		if !engine.stopTokens.IsStopToken(splitStr) {
+			strArr = append(strArr, splitStr)
+		}
+	}
+
+	// Segment 分词
+	if !engine.initOptions.NotUseGse {
+		sehans := engine.Segment(hans)
+		for h := 0; h < len(sehans); h++ {
+			if !engine.stopTokens.IsStopToken(sehans[h]) {
+				strArr = append(strArr, sehans[h])
+			}
+		}
+	}
+	//
+	// py := pinyin.LazyConvert(sehans[h], nil)
+	py := gpy.LazyConvert(hans, nil)
+
+	// log.Println("py...", py)
+	for i := 0; i < len(py); i++ {
+		// log.Println("py[i]...", py[i])
+		pyStr += py[i]
+		if !engine.stopTokens.IsStopToken(pyStr) {
+			strArr = append(strArr, pyStr)
+		}
+
+		if len(py[i]) > 0 {
+			str += py[i][0:1]
+			if !engine.stopTokens.IsStopToken(str) {
+				strArr = append(strArr, str)
+			}
+		}
+	}
+
+	return strArr
+}

vendor/github.com/go-ego/riot/stop_tokens.go

@@ -0,0 +1,57 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package riot
+
+import (
+	"bufio"
+	"log"
+	"os"
+)
+
+// StopTokens stop tokens map
+type StopTokens struct {
+	stopTokens map[string]bool
+}
+
+// Init 从 stopTokenFile 中读入停用词，一个词一行
+// 文档索引建立时会跳过这些停用词
+func (st *StopTokens) Init(stopTokenFile string) {
+	st.stopTokens = make(map[string]bool)
+	if stopTokenFile == "" {
+		return
+	}
+
+	file, err := os.Open(stopTokenFile)
+	if err != nil {
+		log.Fatal("Open stop token file error: ", err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if text != "" {
+			st.stopTokens[text] = true
+		}
+	}
+
+}
+
+// IsStopToken to determine whether to stop token
+func (st *StopTokens) IsStopToken(token string) bool {
+	_, found := st.stopTokens[token]
+	return found
+}

vendor/github.com/go-ego/riot/store/BUILD.bazel

@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "badger_store.go",
+        "bolt_store.go",
+        "ldb_store.go",
+        "store.go",
+    ],
+    importmap = "go-common/vendor/github.com/go-ego/riot/store",
+    importpath = "github.com/go-ego/riot/store",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/coreos/bbolt:go_default_library",
+        "//vendor/github.com/dgraph-io/badger:go_default_library",
+        "//vendor/github.com/syndtr/goleveldb/leveldb:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/go-ego/riot/store/badger_store.go

@@ -0,0 +1,140 @@
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package store
+
+import (
+	"log"
+
+	"github.com/dgraph-io/badger"
+)
+
+// Badger badger.KV db store
+type Badger struct {
+	db *badger.DB
+}
+
+// OpenBadger open Badger store
+func OpenBadger(dbPath string) (Store, error) {
+	// err := os.MkdirAll(dbPath, 0777)
+	// if err != nil {
+	// 	log.Fatal("os.MkdirAll: ", err)
+	// 	os.Exit(1)
+	// }
+	// os.MkdirAll(path.Dir(dbPath), os.ModePerm)
+
+	opt := badger.DefaultOptions
+	opt.Dir = dbPath
+	opt.ValueDir = dbPath
+	opt.SyncWrites = true
+	kv, err := badger.Open(opt)
+	if err != nil {
+		log.Fatal("badger NewKV: ", err)
+	}
+
+	return &Badger{kv}, err
+}
+
+// WALName is useless for this kv database
+func (s *Badger) WALName() string {
+	return "" // 对于此数据库，本函数没用~
+}
+
+// Set sets the provided value for a given key.
+// If key is not present, it is created. If it is present,
+// the existing value is overwritten with the one provided.
+func (s *Badger) Set(k, v []byte) error {
+	err := s.db.Update(func(txn *badger.Txn) error {
+		// return txn.Set(k, v, 0x00)
+		return txn.Set(k, v)
+	})
+
+	return err
+}
+
+// Get looks for key and returns a value.
+// If key is not found, value is nil.
+func (s *Badger) Get(k []byte) ([]byte, error) {
+	var ival []byte
+	err := s.db.View(func(txn *badger.Txn) error {
+		item, err := txn.Get(k)
+		if err != nil {
+			return err
+		}
+
+		ival, err = item.Value()
+		return err
+	})
+
+	return ival, err
+}
+
+// Delete deletes a key. Exposing this so that user does not
+// have to specify the Entry directly. For example, BitDelete
+// seems internal to badger.
+func (s *Badger) Delete(k []byte) error {
+	err := s.db.Update(func(txn *badger.Txn) error {
+		return txn.Delete(k)
+	})
+
+	return err
+}
+
+// Has returns true if the DB does contains the given key.
+func (s *Badger) Has(k []byte) (bool, error) {
+	// return s.db.Exists(k)
+	val, err := s.Get(k)
+	if string(val) == "" && err != nil {
+		return false, err
+	}
+
+	return true, err
+}
+
+// Len returns the size of lsm and value log files in bytes.
+// It can be used to decide how often to call RunValueLogGC.
+func (s *Badger) Len() (int64, int64) {
+	return s.db.Size()
+}
+
+// ForEach get all key and value
+func (s *Badger) ForEach(fn func(k, v []byte) error) error {
+	err := s.db.View(func(txn *badger.Txn) error {
+		opts := badger.DefaultIteratorOptions
+		opts.PrefetchSize = 1000
+		it := txn.NewIterator(opts)
+		defer it.Close()
+		for it.Rewind(); it.Valid(); it.Next() {
+			item := it.Item()
+			key := item.Key()
+			val, err := item.Value()
+			if err != nil {
+				return err
+			}
+
+			if err := fn(key, val); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	return err
+}
+
+// Close closes a KV. It's crucial to call it to ensure
+// all the pending updates make their way to disk.
+func (s *Badger) Close() error {
+	return s.db.Close()
+}

vendor/github.com/go-ego/riot/store/bolt_store.go

@@ -0,0 +1,118 @@
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package store
+
+import (
+	"time"
+
+	"github.com/coreos/bbolt"
+	// "github.com/boltdb/bolt"
+)
+
+var gdocs = []byte("gdocs")
+
+// Bolt bolt store struct
+type Bolt struct {
+	db *bolt.DB
+}
+
+// OpenBolt open Bolt store
+func OpenBolt(dbPath string) (Store, error) {
+	db, err := bolt.Open(dbPath, 0600, &bolt.Options{Timeout: 3600 * time.Second})
+	// db, err := bolt.Open(dbPath, 0600, &bolt.Options{})
+	if err != nil {
+		return nil, err
+	}
+	err = db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists(gdocs)
+		return err
+	})
+	if err != nil {
+		db.Close()
+		return nil, err
+	}
+	return &Bolt{db}, nil
+}
+
+// WALName returns the path to currently open database file.
+func (s *Bolt) WALName() string {
+	return s.db.Path()
+}
+
+// Set executes a function within the context of a read-write managed
+// transaction. If no error is returned from the function then the transaction
+// is committed. If an error is returned then the entire transaction is rolled back.
+// Any error that is returned from the function or returned from the commit is returned
+// from the Update() method.
+func (s *Bolt) Set(k []byte, v []byte) error {
+	return s.db.Update(func(tx *bolt.Tx) error {
+		return tx.Bucket(gdocs).Put(k, v)
+	})
+}
+
+// Get executes a function within the context of a managed read-only transaction.
+// Any error that is returned from the function is returned from the View() method.
+func (s *Bolt) Get(k []byte) (b []byte, err error) {
+	err = s.db.View(func(tx *bolt.Tx) error {
+		b = tx.Bucket(gdocs).Get(k)
+		return nil
+	})
+	return
+}
+
+// Delete deletes a key. Exposing this so that user does not
+// have to specify the Entry directly.
+func (s *Bolt) Delete(k []byte) error {
+	return s.db.Update(func(tx *bolt.Tx) error {
+		return tx.Bucket(gdocs).Delete(k)
+	})
+}
+
+// Has returns true if the DB does contains the given key.
+func (s *Bolt) Has(k []byte) (bool, error) {
+	// return s.db.Exists(k)
+	var b []byte
+	err := s.db.View(func(tx *bolt.Tx) error {
+		b = tx.Bucket(gdocs).Get(k)
+		return nil
+	})
+
+	// b == nil
+	if err != nil || string(b) == "" {
+		return false, err
+	}
+
+	return true, nil
+}
+
+// ForEach get all key and value
+func (s *Bolt) ForEach(fn func(k, v []byte) error) error {
+	return s.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(gdocs)
+		c := b.Cursor()
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			if err := fn(k, v); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+// Close releases all database resources. All transactions
+// must be closed before closing the database.
+func (s *Bolt) Close() error {
+	return s.db.Close()
+}

vendor/github.com/go-ego/riot/store/ldb_store.go

@@ -0,0 +1,107 @@
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package store
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// Leveldb leveldb store
+type Leveldb struct {
+	db *leveldb.DB
+}
+
+// OpenLeveldb opens or creates a DB for the given store. The DB
+// will be created if not exist, unless ErrorIfMissing is true.
+// Also, if ErrorIfExist is true and the DB exist Open will
+// returns os.ErrExist error.
+func OpenLeveldb(dbPath string) (Store, error) {
+	db, err := leveldb.OpenFile(dbPath, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Leveldb{db}, nil
+}
+
+// WALName is useless for this kv database
+func (s *Leveldb) WALName() string {
+	return "" // 对于此数据库，本函数没用~
+}
+
+// Set sets the provided value for a given key.
+// If key is not present, it is created. If it is present,
+// the existing value is overwritten with the one provided.
+func (s *Leveldb) Set(k, v []byte) error {
+	return s.db.Put(k, v, nil)
+}
+
+// Get gets the value for the given key. It returns
+// ErrNotFound if the DB does not contains the key.
+//
+// The returned slice is its own copy, it is safe to modify
+// the contents of the returned slice. It is safe to modify the contents
+// of the argument after Get returns.
+func (s *Leveldb) Get(k []byte) ([]byte, error) {
+	return s.db.Get(k, nil)
+}
+
+// Delete deletes the value for the given key. Delete will not
+// returns error if key doesn't exist. Write merge also applies
+// for Delete, see Write.
+//
+// It is safe to modify the contents of the arguments after Delete
+// returns but not before.
+func (s *Leveldb) Delete(k []byte) error {
+	return s.db.Delete(k, nil)
+}
+
+// Has returns true if the DB does contains the given key.
+// It is safe to modify the contents of the argument after Has returns.
+func (s *Leveldb) Has(k []byte) (bool, error) {
+	return s.db.Has(k, nil)
+}
+
+// Len calculates approximate sizes of the given key ranges.
+// The length of the returned sizes are equal with the length of
+// the given ranges. The returned sizes measure store space usage,
+// so if the user data compresses by a factor of ten, the returned
+// sizes will be one-tenth the size of the corresponding user data size.
+// The results may not include the sizes of recently written data.
+func (s *Leveldb) Len() (leveldb.Sizes, error) {
+	return s.db.SizeOf(nil)
+}
+
+// ForEach get all key and value
+func (s *Leveldb) ForEach(fn func(k, v []byte) error) error {
+	iter := s.db.NewIterator(nil, nil)
+	for iter.Next() {
+		// Remember that the contents of the returned slice should not be modified, and
+		// only valid until the next call to Next.
+		key := iter.Key()
+		val := iter.Value()
+		if err := fn(key, val); err != nil {
+			return err
+		}
+	}
+	iter.Release()
+	return iter.Error()
+}
+
+// Close closes the DB. This will also releases any outstanding snapshot,
+// abort any in-flight compaction and discard open transaction.
+func (s *Leveldb) Close() error {
+	return s.db.Close()
+}

vendor/github.com/go-ego/riot/store/store.go

@@ -0,0 +1,72 @@
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package store
+
+import (
+	"fmt"
+	"os"
+)
+
+const (
+	// DefaultStore default store engine
+	DefaultStore = "ldb"
+	// DefaultStore = "bad"
+	// DefaultStore = "bolt"
+)
+
+var supportedStore = map[string]func(path string) (Store, error){
+	"ldb":  OpenLeveldb,
+	"bg":   OpenBadger, // bad to bg
+	"bolt": OpenBolt,
+	// "kv":   OpenKV,
+	// "ledisdb": Open,
+}
+
+// RegisterStore register store engine
+func RegisterStore(name string, fn func(path string) (Store, error)) {
+	supportedStore[name] = fn
+}
+
+// Store is store interface
+type Store interface {
+	// type KVBatch interface {
+	Set(k, v []byte) error
+	Get(k []byte) ([]byte, error)
+	Delete(k []byte) error
+	Has(k []byte) (bool, error)
+	ForEach(fn func(k, v []byte) error) error
+	Close() error
+	WALName() string
+}
+
+// OpenStore open store engine
+func OpenStore(path string, args ...string) (Store, error) {
+	storeName := DefaultStore
+
+	if len(args) > 0 && args[0] != "" {
+		storeName = args[0]
+	} else {
+		storeEnv := os.Getenv("Riot_Store_Engine")
+		if storeEnv != "" {
+			storeName = storeEnv
+		}
+	}
+
+	if fn, has := supportedStore[storeName]; has {
+		return fn(path)
+	}
+
+	return nil, fmt.Errorf("unsupported store engine: %v", storeName)
+}

vendor/github.com/go-ego/riot/store_worker.go

@@ -0,0 +1,97 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package riot
+
+import (
+	"bytes"
+
+	"encoding/binary"
+	"encoding/gob"
+	"sync/atomic"
+
+	"github.com/go-ego/riot/types"
+)
+
+type storeIndexDocReq struct {
+	docId uint64
+	data  types.DocData
+	// data        types.DocumentIndexData
+}
+
+func (engine *Engine) storeIndexDocWorker(shard int) {
+	for {
+		request := <-engine.storeIndexDocChans[shard]
+
+		// 得到 key
+		b := make([]byte, 10)
+		length := binary.PutUvarint(b, request.docId)
+
+		// 得到 value
+		var buf bytes.Buffer
+		enc := gob.NewEncoder(&buf)
+		err := enc.Encode(request.data)
+		if err != nil {
+			atomic.AddUint64(&engine.numDocsStored, 1)
+			continue
+		}
+
+		// has, err := engine.dbs[shard].Has(b[0:length])
+		// if err != nil {
+		// 	log.Println("engine.dbs[shard].Has(b[0:length]) ", err)
+		// }
+
+		// if has {
+		// 	engine.dbs[shard].Delete(b[0:length])
+		// }
+
+		// 将 key-value 写入数据库
+		engine.dbs[shard].Set(b[0:length], buf.Bytes())
+
+		engine.loc.Lock()
+		atomic.AddUint64(&engine.numDocsStored, 1)
+		engine.loc.Unlock()
+	}
+}
+
+func (engine *Engine) storeRemoveDocWorker(docId uint64, shard uint32) {
+	// 得到 key
+	b := make([]byte, 10)
+	length := binary.PutUvarint(b, docId)
+
+	// 从数据库删除该 key
+	engine.dbs[shard].Delete(b[0:length])
+}
+
+// storageInitWorker persistent storage init worker
+func (engine *Engine) storeInitWorker(shard int) {
+	engine.dbs[shard].ForEach(func(k, v []byte) error {
+		key, value := k, v
+		// 得到 docID
+		docId, _ := binary.Uvarint(key)
+
+		// 得到 data
+		buf := bytes.NewReader(value)
+		dec := gob.NewDecoder(buf)
+		var data types.DocData
+		err := dec.Decode(&data)
+		if err == nil {
+			// 添加索引
+			engine.internalIndexDoc(docId, data, false)
+		}
+		return nil
+	})
+	engine.storeInitChan <- true
+}

vendor/github.com/go-ego/riot/types/BUILD.bazel

@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc_index_data.go",
+        "doc_info.go",
+        "engine_init_options.go",
+        "index.go",
+        "indexer_init_options.go",
+        "scoring_criteria.go",
+        "search_request.go",
+        "search_response.go",
+    ],
+    importmap = "go-common/vendor/github.com/go-ego/riot/types",
+    importpath = "github.com/go-ego/riot/types",
+    visibility = ["//visibility:public"],
+    deps = ["//vendor/github.com/go-ego/riot/utils:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/go-ego/riot/types/doc_index_data.go

@@ -0,0 +1,62 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package types
+
+// DocIndexData type document Index Data struct
+// type DocIndexData DocData
+type DocIndexData = DocData
+
+// DocData type document Index Data struct
+type DocData struct {
+	// 文档全文（必须是 UTF-8 格式），用于生成待索引的关键词
+	Content string
+
+	// new 类别
+	// Class string
+	// new 属性
+	Attri interface{}
+
+	// 文档的关键词
+	// 当 Content 不为空的时候，优先从 Content 中分词得到关键词。
+	// Tokens 存在的意义在于绕过 riot 内置的分词器，在引擎外部
+	// 进行分词和预处理。
+	// Tokens []*TokenData
+	Tokens []TokenData
+
+	// 文档标签（必须是 UTF-8 格式），比如文档的类别属性等，
+	// 这些标签并不出现在文档文本中
+	Labels []string
+
+	// 文档的评分字段，可以接纳任何类型的结构体
+	Fields interface{}
+}
+
+// TokenData 文档的一个关键词
+type TokenData struct {
+	// 关键词的字符串
+	Text string
+
+	// 关键词的首字节在文档中出现的位置
+	Locations []int
+}
+
+// Attri doc attribute
+type Attri struct {
+	Title  string `json:"title"`
+	Author string `json:"author"`
+	Time   string `json:"time"`
+	Ts     int64  `json:"ts"`
+}

vendor/github.com/go-ego/riot/types/doc_info.go

@@ -0,0 +1,51 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package types
+
+import (
+	"sync"
+)
+
+// DocInfosShard 文档信息[id]info
+type DocInfosShard struct {
+	DocInfos map[uint64]*DocInfo
+	NumDocs  uint64 // 这实际上是总文档数的一个近似
+	sync.RWMutex
+}
+
+// DocInfo document info
+type DocInfo struct {
+	Fields    interface{}
+	TokenLens float32
+}
+
+/// inverted_index.go
+
+// InvertedIndexShard 反向索引表([关键词]反向索引表)
+type InvertedIndexShard struct {
+	InvertedIndex map[string]*KeywordIndices
+	TotalTokenLen float32 //总关键词数
+	sync.RWMutex
+}
+
+// KeywordIndices 反向索引表的一行，收集了一个搜索键出现的所有文档，
+// 按照 DocId 从小到大排序。
+type KeywordIndices struct {
+	// 下面的切片是否为空，取决于初始化时 IndexType 的值
+	DocIds      []uint64  // 全部类型都有
+	Frequencies []float32 // IndexType == FrequenciesIndex
+	Locations   [][]int   // IndexType == LocsIndex
+}

vendor/github.com/go-ego/riot/types/engine_init_options.go

@@ -0,0 +1,158 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package types
+
+import (
+	"runtime"
+)
+
+var (
+	// EngineOpts 的默认值
+	// defaultNumGseThreads default Segmenter threads num
+	defaultNumGseThreads = runtime.NumCPU()
+	// defaultNumShards                 = 2
+	defaultNumShards                 = 8
+	defaultIndexerBufLen             = runtime.NumCPU()
+	defaultNumIndexerThreadsPerShard = runtime.NumCPU()
+	defaultRankerBufLen              = runtime.NumCPU()
+	defaultNumRankerThreadsPerShard  = runtime.NumCPU()
+	defaultDefaultRankOpts           = RankOpts{
+		ScoringCriteria: RankByBM25{},
+	}
+	defaultIndexerOpts = IndexerOpts{
+		IndexType:      FrequenciesIndex,
+		BM25Parameters: &defaultBM25Parameters,
+	}
+	defaultBM25Parameters = BM25Parameters{
+		K1: 2.0,
+		B:  0.75,
+	}
+	defaultStoreShards = 8
+)
+
+// EngineOpts init engine options
+type EngineOpts struct {
+	// 是否使用分词器
+	// 默认使用，否则在启动阶段跳过 GseDict 和 StopTokenFile 设置
+	// 如果你不需要在引擎内分词，可以将这个选项设为 true
+	// 注意，如果你不用分词器，那么在调用 IndexDoc 时,
+	// DocIndexData 中的 Content 会被忽略
+	// Not use the gse segment
+	NotUseGse bool `toml:"not_use_gse"`
+
+	// new, 分词规则
+	Using int `toml:"using"`
+
+	// 半角逗号 "," 分隔的字典文件，具体用法见
+	// gse.Segmenter.LoadDict 函数的注释
+	GseDict string `toml:"gse_dict"`
+	// GseDict []string
+	SegmenterDict string
+
+	// 停用词文件
+	StopTokenFile string `toml:"stop_file"`
+	// Gse search mode
+	GseMode bool `toml:"gse_mode"`
+
+	// 分词器线程数
+	// NumSegmenterThreads int
+	NumGseThreads int
+
+	// 索引器和排序器的 shard 数目
+	// 被检索/排序的文档会被均匀分配到各个 shard 中
+	NumShards int
+
+	// 索引器的信道缓冲长度
+	IndexerBufLen int
+
+	// 索引器每个shard分配的线程数
+	NumIndexerThreadsPerShard int
+
+	// 排序器的信道缓冲长度
+	RankerBufLen int
+
+	// 排序器每个 shard 分配的线程数
+	NumRankerThreadsPerShard int
+
+	// 索引器初始化选项
+	IndexerOpts *IndexerOpts
+
+	// 默认的搜索选项
+	DefaultRankOpts *RankOpts
+
+	// 是否使用持久数据库，以及数据库文件保存的目录和裂分数目
+	StoreOnly bool `toml:"store_only"`
+	UseStore  bool `toml:"use_store"`
+
+	StoreFolder string `toml:"store_folder"`
+	StoreShards int    `toml:"store_shards"`
+	StoreEngine string `toml:"store_engine"`
+
+	IDOnly bool `toml:"id_only"`
+}
+
+// Init init engine options
+// 初始化 EngineOpts，当用户未设定某个选项的值时用默认值取代
+func (options *EngineOpts) Init() {
+	// if !options.NotUseGse && options.GseDict == "" {
+	// 	log.Fatal("字典文件不能为空")
+	//  options.GseDict = "zh"
+	// }
+
+	if options.NumGseThreads == 0 {
+		options.NumGseThreads = defaultNumGseThreads
+	}
+
+	if options.NumShards == 0 {
+		options.NumShards = defaultNumShards
+	}
+
+	if options.IndexerBufLen == 0 {
+		options.IndexerBufLen = defaultIndexerBufLen
+	}
+
+	if options.NumIndexerThreadsPerShard == 0 {
+		options.NumIndexerThreadsPerShard = defaultNumIndexerThreadsPerShard
+	}
+
+	if options.RankerBufLen == 0 {
+		options.RankerBufLen = defaultRankerBufLen
+	}
+
+	if options.NumRankerThreadsPerShard == 0 {
+		options.NumRankerThreadsPerShard = defaultNumRankerThreadsPerShard
+	}
+
+	if options.IndexerOpts == nil {
+		options.IndexerOpts = &defaultIndexerOpts
+	}
+
+	if options.IndexerOpts.BM25Parameters == nil {
+		options.IndexerOpts.BM25Parameters = &defaultBM25Parameters
+	}
+
+	if options.DefaultRankOpts == nil {
+		options.DefaultRankOpts = &defaultDefaultRankOpts
+	}
+
+	if options.DefaultRankOpts.ScoringCriteria == nil {
+		options.DefaultRankOpts.ScoringCriteria = defaultDefaultRankOpts.ScoringCriteria
+	}
+
+	if options.StoreShards == 0 {
+		options.StoreShards = defaultStoreShards
+	}
+}

vendor/github.com/go-ego/riot/types/index.go

@@ -0,0 +1,97 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+/*
+
+Package types is riot types
+*/
+package types
+
+// DocIndex document's index
+type DocIndex struct {
+	// DocId 文本的 DocId
+	DocId uint64
+
+	// TokenLen 文本的关键词长
+	TokenLen float32
+
+	// Keywords 加入的索引键
+	Keywords []KeywordIndex
+}
+
+// KeywordIndex 反向索引项，这实际上标注了一个（搜索键，文档）对。
+type KeywordIndex struct {
+	// Text 搜索键的 UTF-8 文本
+	Text string
+
+	// Frequency 搜索键词频
+	Frequency float32
+
+	// Starts 搜索键在文档中的起始字节位置，按照升序排列
+	Starts []int
+}
+
+// IndexedDoc 索引器返回结果
+type IndexedDoc struct {
+	// DocId document id
+	DocId uint64
+
+	// BM25，仅当索引类型为 FrequenciesIndex 或者 LocsIndex 时返回有效值
+	BM25 float32
+
+	// TokenProximity 关键词在文档中的紧邻距离，
+	// 紧邻距离的含义见 computeTokenProximity 的注释。
+	// 仅当索引类型为 LocsIndex 时返回有效值。
+	TokenProximity int32
+
+	// TokenSnippetLocs 紧邻距离计算得到的关键词位置，
+	// 和 Lookup 函数输入 tokens 的长度一样且一一对应。
+	// 仅当索引类型为 LocsIndex 时返回有效值。
+	TokenSnippetLocs []int
+
+	// TokenLocs 关键词在文本中的具体位置。
+	// 仅当索引类型为 LocsIndex 时返回有效值。
+	TokenLocs [][]int
+}
+
+// DocsIndex 方便批量加入文档索引
+type DocsIndex []*DocIndex
+
+func (docs DocsIndex) Len() int {
+	return len(docs)
+}
+
+func (docs DocsIndex) Swap(i, j int) {
+	docs[i], docs[j] = docs[j], docs[i]
+}
+
+func (docs DocsIndex) Less(i, j int) bool {
+	return docs[i].DocId < docs[j].DocId
+}
+
+// DocsId 方便批量删除文档索引
+type DocsId []uint64
+
+func (docs DocsId) Len() int {
+	return len(docs)
+}
+
+func (docs DocsId) Swap(i, j int) {
+	docs[i], docs[j] = docs[j], docs[i]
+}
+
+func (docs DocsId) Less(i, j int) bool {
+	return docs[i] < docs[j]
+}

vendor/github.com/go-ego/riot/types/indexer_init_options.go

@@ -0,0 +1,58 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package types
+
+// 这些常数定义了反向索引表存储的数据类型
+const (
+	// DocIdsIndex 仅存储文档的 docId
+	DocIdsIndex = 0
+
+	// FrequenciesIndex 存储关键词的词频，用于计算BM25
+	FrequenciesIndex = 1
+
+	// LocsIndex 存储关键词在文档中出现的具体字节位置（可能有多个）
+	// 如果你希望得到关键词紧邻度数据，必须使用 LocsIndex 类型的索引
+	LocsIndex = 2
+
+	// 默认插入索引表文档 CACHE SIZE
+	defaultDocCacheSize = 300000
+)
+
+// IndexerOpts 初始化索引器选项
+type IndexerOpts struct {
+	// 索引表的类型，见上面的常数
+	IndexType int
+
+	// 待插入索引表文档 CACHE SIZE
+	DocCacheSize int
+
+	// BM25 参数
+	BM25Parameters *BM25Parameters
+}
+
+// BM25Parameters 见http://en.wikipedia.org/wiki/Okapi_BM25
+// 默认值见 engine_init_options.go
+type BM25Parameters struct {
+	K1 float32
+	B  float32
+}
+
+// Init init IndexerOpts
+func (options *IndexerOpts) Init() {
+	if options.DocCacheSize == 0 {
+		options.DocCacheSize = defaultDocCacheSize
+	}
+}

vendor/github.com/go-ego/riot/types/scoring_criteria.go

@@ -0,0 +1,33 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package types
+
+// ScoringCriteria 评分规则通用接口
+type ScoringCriteria interface {
+	// 给一个文档评分，文档排序时先用第一个分值比较，如果
+	// 分值相同则转移到第二个分值，以此类推。
+	// 返回空切片表明该文档应该从最终排序结果中剔除。
+	Score(doc IndexedDoc, fields interface{}) []float32
+}
+
+// RankByBM25 一个简单的评分规则，文档分数为BM25
+type RankByBM25 struct {
+}
+
+// Score score
+func (rule RankByBM25) Score(doc IndexedDoc, fields interface{}) []float32 {
+	return []float32{doc.BM25}
+}

vendor/github.com/go-ego/riot/types/search_request.go

@@ -0,0 +1,99 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package types
+
+// SearchReq search request options
+type SearchReq struct {
+	// 搜索的短语（必须是 UTF-8 格式），会被分词
+	// 当值为空字符串时关键词会从下面的 Tokens 读入
+	Text string
+
+	// 关键词（必须是 UTF-8 格式），当 Text 不为空时优先使用 Text
+	// 通常你不需要自己指定关键词，除非你运行自己的分词程序
+	Tokens []string
+
+	// 文档标签（必须是 UTF-8 格式），标签不存在文档文本中，
+	// 但也属于搜索键的一种
+	Labels []string
+
+	// 类别
+	// Class string
+
+	// 逻辑检索表达式
+	Logic Logic
+
+	// 当不为 nil 时，仅从这些 DocIds 包含的键中搜索（忽略值）
+	DocIds map[uint64]bool
+
+	// 排序选项
+	RankOpts *RankOpts
+
+	// 超时，单位毫秒（千分之一秒）。此值小于等于零时不设超时。
+	// 搜索超时的情况下仍有可能返回部分排序结果。
+	Timeout int
+
+	// 设为 true 时仅统计搜索到的文档个数，不返回具体的文档
+	CountDocsOnly bool
+
+	// 不排序，对于可在引擎外部（比如客户端）排序情况适用
+	// 对返回文档很多的情况打开此选项可以有效节省时间
+	Orderless bool
+}
+
+// RankOpts rank options
+type RankOpts struct {
+	// 文档的评分规则，值为 nil 时使用 Engine 初始化时设定的规则
+	ScoringCriteria ScoringCriteria
+
+	// 默认情况下（ReverseOrder = false）按照分数从大到小排序，否则从小到大排序
+	ReverseOrder bool
+
+	// 从第几条结果开始输出
+	OutputOffset int
+
+	// 最大输出的搜索结果数，为 0 时无限制
+	MaxOutputs int
+}
+
+// Logic logic options
+type Logic struct {
+	// return all doc
+	// All bool
+
+	// 与查询, 必须都存在
+	Must bool
+
+	// 或查询, 有一个存在即可
+	Should bool
+
+	// 非查询, 不包含
+	NotIn bool
+
+	LogicExpr LogicExpr
+}
+
+// LogicExpr logic expression options
+type LogicExpr struct {
+
+	// 与查询, 必须都存在
+	MustLabels []string
+
+	// 或查询, 有一个存在即可
+	ShouldLabels []string
+
+	// 非查询, 不包含
+	NotInLabels []string
+}

vendor/github.com/go-ego/riot/types/search_response.go

@@ -0,0 +1,152 @@
+// Copyright 2013 Hui Chen
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package types
+
+import (
+	"github.com/go-ego/riot/utils"
+)
+
+// SearchResp search response options
+type SearchResp struct {
+	// 搜索用到的关键词
+	Tokens []string
+
+	// 类别
+	// Class string
+
+	// 搜索到的文档，已排序
+	// Docs []ScoredDoc
+	Docs interface{}
+
+	// 搜索是否超时。超时的情况下也可能会返回部分结果
+	Timeout bool
+
+	// 搜索到的文档个数。注意这是全部文档中满足条件的个数，可能比返回的文档数要大
+	NumDocs int
+}
+
+// Content search content
+type Content struct {
+	// new Content
+	Content string
+
+	// new 属性 Attri
+	Attri interface{}
+
+	// new 返回评分字段
+	Fields interface{}
+}
+
+// ScoredDoc scored the document
+type ScoredDoc struct {
+	DocId uint64
+
+	// new 返回文档 Content
+	Content string
+	// new 返回文档属性 Attri
+	Attri interface{}
+	// new 返回评分字段
+	Fields interface{}
+
+	// 文档的打分值
+	// 搜索结果按照 Scores 的值排序，先按照第一个数排，
+	// 如果相同则按照第二个数排序，依次类推。
+	Scores []float32
+
+	// 用于生成摘要的关键词在文本中的字节位置，
+	// 该切片长度和 SearchResp.Tokens 的长度一样
+	// 只有当 IndexType == LocsIndex 时不为空
+	TokenSnippetLocs []int
+
+	// 关键词出现的位置
+	// 只有当 IndexType == LocsIndex 时不为空
+	TokenLocs [][]int
+}
+
+// ScoredDocs 为了方便排序
+type ScoredDocs []ScoredDoc
+
+func (docs ScoredDocs) Len() int {
+	return len(docs)
+}
+
+func (docs ScoredDocs) Swap(i, j int) {
+	docs[i], docs[j] = docs[j], docs[i]
+}
+
+func (docs ScoredDocs) Less(i, j int) bool {
+	// 为了从大到小排序，这实际上实现的是 More 的功能
+	for iScore := 0; iScore < utils.MinInt(len(docs[i].Scores), len(docs[j].Scores)); iScore++ {
+		if docs[i].Scores[iScore] > docs[j].Scores[iScore] {
+			return true
+		} else if docs[i].Scores[iScore] < docs[j].Scores[iScore] {
+			return false
+		}
+	}
+	return len(docs[i].Scores) > len(docs[j].Scores)
+}
+
+/*
+  ______   .__   __.  __      ____    ____  __   _______
+ /  __  \  |  \ |  | |  |     \   \  /   / |  | |       \
+|  |  |  | |   \|  | |  |      \   \/   /  |  | |  .--.  |
+|  |  |  | |  . `  | |  |       \_    _/   |  | |  |  |  |
+|  `--'  | |  |\   | |  `----.    |  |     |  | |  '--'  |
+ \______/  |__| \__| |_______|    |__|     |__| |_______/
+
+*/
+
+// ScoredID scored doc only id
+type ScoredID struct {
+	DocId uint64
+
+	// 文档的打分值
+	// 搜索结果按照 Scores 的值排序，先按照第一个数排，
+	// 如果相同则按照第二个数排序，依次类推。
+	Scores []float32
+
+	// 用于生成摘要的关键词在文本中的字节位置，
+	// 该切片长度和 SearchResp.Tokens 的长度一样
+	// 只有当 IndexType == LocsIndex 时不为空
+	TokenSnippetLocs []int
+
+	// 关键词出现的位置
+	// 只有当 IndexType == LocsIndex 时不为空
+	TokenLocs [][]int
+}
+
+// ScoredIDs 为了方便排序
+type ScoredIDs []ScoredID
+
+func (docs ScoredIDs) Len() int {
+	return len(docs)
+}
+
+func (docs ScoredIDs) Swap(i, j int) {
+	docs[i], docs[j] = docs[j], docs[i]
+}
+
+func (docs ScoredIDs) Less(i, j int) bool {
+	// 为了从大到小排序，这实际上实现的是 More 的功能
+	for iScore := 0; iScore < utils.MinInt(len(docs[i].Scores), len(docs[j].Scores)); iScore++ {
+		if docs[i].Scores[iScore] > docs[j].Scores[iScore] {
+			return true
+		} else if docs[i].Scores[iScore] < docs[j].Scores[iScore] {
+			return false
+		}
+	}
+	return len(docs[i].Scores) > len(docs[j].Scores)
+}

vendor/github.com/go-ego/riot/utils/BUILD.bazel

@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "test_utils.go",
+        "utils.go",
+    ],
+    importmap = "go-common/vendor/github.com/go-ego/riot/utils",
+    importpath = "github.com/go-ego/riot/utils",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/go-ego/riot/utils/test_utils.go

@@ -0,0 +1 @@
+package utils

vendor/github.com/go-ego/riot/utils/utils.go

@@ -0,0 +1,31 @@
+// Copyright 2016 ego authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+// AbsInt return to the opposite number
+func AbsInt(a int) int {
+	if a < 0 {
+		return -a
+	}
+	return a
+}
+
+// MinInt return to the small number
+func MinInt(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}