From 99c0b949f7ba15f677de07e935a9a5aea0af4910 Mon Sep 17 00:00:00 2001 From: Griatch Date: Sat, 23 May 2020 14:59:12 +0200 Subject: [PATCH] Move to custom search plugin by borrowing from Mkdocs and existing plugins --- docs/search/README.md | 11 + docs/search/__init__.py | 173 + docs/search/css/searchbox.css | 152 + docs/search/js/search/lunr.js | 3475 +++++++++++++++++ docs/search/js/search/main.js | 100 + docs/search/js/search/search_index.json | 1 + docs/search/js/search/worker.js | 132 + docs/search/lunr-language/lunr.ar.js | 381 ++ docs/search/lunr-language/lunr.da.js | 284 ++ docs/search/lunr-language/lunr.de.js | 384 ++ docs/search/lunr-language/lunr.du.js | 450 +++ docs/search/lunr-language/lunr.es.js | 599 +++ docs/search/lunr-language/lunr.fi.js | 541 +++ docs/search/lunr-language/lunr.fr.js | 703 ++++ docs/search/lunr-language/lunr.hu.js | 565 +++ docs/search/lunr-language/lunr.it.js | 617 +++ docs/search/lunr-language/lunr.ja.js | 188 + docs/search/lunr-language/lunr.jp.js | 5 + docs/search/lunr-language/lunr.multi.js | 79 + docs/search/lunr-language/lunr.nl.js | 448 +++ docs/search/lunr-language/lunr.no.js | 258 ++ docs/search/lunr-language/lunr.pt.js | 570 +++ docs/search/lunr-language/lunr.ro.js | 558 +++ docs/search/lunr-language/lunr.ru.js | 391 ++ .../lunr-language/lunr.stemmer.support.js | 304 ++ docs/search/lunr-language/lunr.sv.js | 256 ++ docs/search/lunr-language/lunr.th.js | 97 + docs/search/lunr-language/lunr.tr.js | 1087 ++++++ docs/search/lunr-language/lunr.vi.js | 84 + docs/search/searchbox.html | 13 + docs/source/conf.py | 18 +- 31 files changed, 12915 insertions(+), 9 deletions(-) create mode 100644 docs/search/README.md create mode 100644 docs/search/__init__.py create mode 100644 docs/search/css/searchbox.css create mode 100644 docs/search/js/search/lunr.js create mode 100644 docs/search/js/search/main.js create mode 100644 docs/search/js/search/search_index.json create mode 100644 docs/search/js/search/worker.js create mode 100644 docs/search/lunr-language/lunr.ar.js create mode 100644 docs/search/lunr-language/lunr.da.js create mode 100644 docs/search/lunr-language/lunr.de.js create mode 100644 docs/search/lunr-language/lunr.du.js create mode 100644 docs/search/lunr-language/lunr.es.js create mode 100644 docs/search/lunr-language/lunr.fi.js create mode 100644 docs/search/lunr-language/lunr.fr.js create mode 100644 docs/search/lunr-language/lunr.hu.js create mode 100644 docs/search/lunr-language/lunr.it.js create mode 100644 docs/search/lunr-language/lunr.ja.js create mode 100644 docs/search/lunr-language/lunr.jp.js create mode 100644 docs/search/lunr-language/lunr.multi.js create mode 100644 docs/search/lunr-language/lunr.nl.js create mode 100644 docs/search/lunr-language/lunr.no.js create mode 100644 docs/search/lunr-language/lunr.pt.js create mode 100644 docs/search/lunr-language/lunr.ro.js create mode 100644 docs/search/lunr-language/lunr.ru.js create mode 100644 docs/search/lunr-language/lunr.stemmer.support.js create mode 100644 docs/search/lunr-language/lunr.sv.js create mode 100644 docs/search/lunr-language/lunr.th.js create mode 100644 docs/search/lunr-language/lunr.tr.js create mode 100644 docs/search/lunr-language/lunr.vi.js create mode 100644 docs/search/searchbox.html diff --git a/docs/search/README.md b/docs/search/README.md new file mode 100644 index 0000000000..1d713230aa --- /dev/null +++ b/docs/search/README.md @@ -0,0 +1,11 @@ +# Search plugin + +This is a search plugin that modifies and combines code from two projects: + +- Sphinxcontrib-lunrsearch (https://github.com/rmcgibbo/sphinxcontrib-lunrsearch) - the lunr-index + generation component was reworked to read more than just the api and addded supoort for + using the `lunr` python library to pre-generate the search index. +- Mkdocs lunr search (https://www.mkdocs.org/) - the javascript components with + web-worker support, using the index generated by the above module. + +Parsing of the documentation to create the index is custom. diff --git a/docs/search/__init__.py b/docs/search/__init__.py new file mode 100644 index 0000000000..52da2d45c5 --- /dev/null +++ b/docs/search/__init__.py @@ -0,0 +1,173 @@ +""" +Custom Evennia search plugin. This combines code from sphinxplugin-lunr and the +Mkdocs search implementation. + +""" +from os.path import dirname, join, exists +from os import makedirs +import json +import sphinx.search +from six import iteritems +from sphinx.util.osutil import copyfile +from sphinx.jinja2glue import SphinxFileSystemLoader + + +# Sphinx setup + +lunr = None +try: + import lunr +except ImportError: + pass + + +def _make_iter(inp): + """make sure input is an iterable""" + if not hasattr(inp, "__iter__"): + return (inp, ) + return inp + + +class IndexBuilder(sphinx.search.IndexBuilder): + def freeze(self): + """Create a usable data structure for serializing.""" + data = super(IndexBuilder, self).freeze() + try: + # Sphinx >= 1.5 format + # Due to changes from github.com/sphinx-doc/sphinx/pull/2454 + base_file_names = data['docnames'] + except KeyError: + # Sphinx < 1.5 format + base_file_names = data['filenames'] + + lunrdocuments = [] + for prefix, items in iteritems(data['objects']): + # This parses API objects + for name, (index, typeindex, _, shortanchor) in iteritems(items): + objtype = data['objtypes'][typeindex] + + if objtype.startswith("py:"): + # Python API entitites + last_prefix = prefix.split('.')[-1] + if objtype == "py:method": + displayname = last_prefix + "." + name + else: + displayname = prefix + "." + name + + else: + last_prefix = prefix.split('.')[-1] + displayname = name + + anchor = f"#{shortanchor}" if shortanchor else '' + lunrdocuments.append({ + 'location': base_file_names[index] + anchor, + 'title': displayname, + 'text': prefix + }) + + titles = data['titles'] + for titleterm, indices in data['titleterms'].items(): + # Title components; the indices map to index in base_file_name + for index in _make_iter(indices): + lunrdocuments.append({ + 'location': base_file_names[index], + 'title': titles[index], + 'text': titleterm + }) + + # this is just too big for regular use + # for term, indices in data['terms'].items(): + # # In-file terms + # for index in _make_iter(indices): + # ref = next(c) + # lunrdocuments[ref] = { + # 'ref': str(ref), + # 'filename': base_file_names[index], + # 'objtype': "", + # 'prefix': term, + # 'last_prefix': '', + # 'name': titles[index], + # 'displayname': titles[index], + # 'shortanchor': '' + # } + + if not lunr: + print("\npython package `lunr==0.5.8` required in order " + "to pre-build search index.") + return data + + print("\nPre-building search index using python-lunr ...") + # pre-compile the data store into a lunr index + fields = ["location", "title", "text"] + lunr_index = lunr.lunr(ref='location', fields=fields, + documents=lunrdocuments) + lunr_index = lunr_index.serialize() + + # required by js file + page_store = { + "config": {"lang": ['en'], + "separator": r'\s\-]+', + "min_search_length": 3, + "prebuild_index": "python"}, + "docs": lunrdocuments, + "index": lunr_index + } + + lunr_index_json = json.dumps(page_store, sort_keys=True, + separators=(',', ':')) + try: + fname = join(dirname(__file__), "js", "search", "search_index.json") + with open(fname, 'w') as fil: + fil.write(lunr_index_json) + except Exception as err: + print("Failed saving lunr index to", fname, err) + + return data + + +def builder_inited(app): + """ + Adding a new loader to the template system puts our searchbox.html + template in front of the others, it overrides whatever searchbox.html + the current theme is using. + it's still up to the theme to actually _use_ a file called searchbox.html + somewhere in its layout. but the base theme and pretty much everything + else that inherits from it uses this filename. + """ + app.builder.templates.loaders.insert(0, SphinxFileSystemLoader(dirname(__file__))) + + +def copy_static_files(app, _): + """ + Because we're using the extension system instead of the theme system, it's + our responsibility to copy over static files outselves. files = + [join('js', 'searchbox.js'), join('css', 'searchbox.css')] + """ + files = [join('js', 'search', 'main.js'), + join('js', 'search', 'worker.js'), + join('js', 'search', 'lunr.js')] + + if lunr: + files.append(join('js', 'search', "search_index.json")) + + for f in files: + src = join(dirname(__file__), f) + dest = join(app.outdir, '_static', f) + if not exists(dirname(dest)): + makedirs(dirname(dest)) + copyfile(src, dest) + + +def setup(app): + # adds