indexer

package

v0.7.0 Latest Latest Go to latest Published: Mar 5, 2026 License: AGPL-3.0 Imports: 61 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/asciimoo/hister

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
func Add(d *Document) error
func Delete(u string) error
func DetectLanguage(s string) (*lingua.Language, error)
func DocumentCount() uint64
func Extract(d *Document) error
func Init(cfg *config.Config) error
func Iterate(fn func(*Document))
func Reindex(basePath string, rules *config.Rules, skipSensitiveChecks bool) error
type Document
- func GetByURL(u string) *Document
- func (d *Document) DownloadFavicon(userAgent string) error
- func (d *Document) Process() error
type Extractor
type Query
- func (q *Query) ToJSON() []byte
type Results
- func Search(cfg *config.Config, q *Query) (*Results, error)

Constants ¶

View Source

const UnknownLanguage = "unknown"

Variables ¶

View Source

var ErrNoExtractor = errors.New("no extractor found")

View Source

var (
	ErrSensitiveContent = errors.New("document contains sensitive data")
)

View Source

var Languages = []lingua.Language{
	lingua.Arabic,
	lingua.Bulgarian,
	lingua.Catalan,
	lingua.Czech,
	lingua.Danish,
	lingua.German,
	lingua.Greek,
	lingua.English,
	lingua.Spanish,
	lingua.Basque,
	lingua.Persian,
	lingua.Finnish,
	lingua.French,
	lingua.Irish,
	lingua.Hindi,
	lingua.Croatian,
	lingua.Hungarian,
	lingua.Armenian,
	lingua.Indonesian,
	lingua.Italian,
	lingua.Dutch,
	lingua.Polish,
	lingua.Portuguese,
	lingua.Romanian,
	lingua.Russian,
	lingua.Swedish,
	lingua.Turkish,
}

View Source

var Version = 2

Functions ¶

func Add ¶

func Add(d *Document) error

func Delete ¶ added in v0.4.0

func Delete(u string) error

func DetectLanguage ¶ added in v0.6.0

func DetectLanguage(s string) (*lingua.Language, error)

func DocumentCount ¶ added in v0.6.0

func DocumentCount() uint64

func Extract ¶ added in v0.4.0

func Extract(d *Document) error

func Init ¶

func Init(cfg *config.Config) error

func Iterate ¶

func Iterate(fn func(*Document))

func Reindex ¶ added in v0.3.0

func Reindex(basePath string, rules *config.Rules, skipSensitiveChecks bool) error

Types ¶

type Document ¶

type Document struct {
	URL      string  `json:"url"`
	Domain   string  `json:"domain"`
	HTML     string  `json:"html"`
	Title    string  `json:"title"`
	Text     string  `json:"text"`
	Favicon  string  `json:"favicon"`
	Score    float64 `json:"score"`
	Added    int64   `json:"added"`
	Language string  `json:"language"`
	// contains filtered or unexported fields
}

func GetByURL ¶ added in v0.2.0

func GetByURL(u string) *Document

func (*Document) DownloadFavicon ¶

func (d *Document) DownloadFavicon(userAgent string) error

func (*Document) Process ¶

func (d *Document) Process() error

type Extractor ¶ added in v0.4.0

type Extractor interface {
	Name() string
	Match(*Document) bool
	Extract(*Document) error
}

type Query ¶

type Query struct {
	Text      string `json:"text"`
	Highlight string `json:"highlight"`
	Limit     int    `json:"limit"`
	Sort      string `json:"sort"`
	DateFrom  int64  `json:"date_from"`
	DateTo    int64  `json:"date_to"`
	// contains filtered or unexported fields
}

func (*Query) ToJSON ¶ added in v0.4.0

func (q *Query) ToJSON() []byte

type Results ¶

type Results struct {
	Total           uint64            `json:"total"`
	Query           *Query            `json:"query"`
	Documents       []*Document       `json:"documents"`
	History         []*model.URLCount `json:"history"`
	SearchDuration  string            `json:"search_duration"`
	QuerySuggestion string            `json:"query_suggestion"`
}

func Search ¶

func Search(cfg *config.Config, q *Query) (*Results, error)

Source Files ¶

View all Source files

Directories ¶

Path	Synopsis
querybuilder

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL