diff --git a/_elastic/indexer.py b/_elastic/indexer.py new file mode 100644 index 000000000000..5c758a727e9a --- /dev/null +++ b/_elastic/indexer.py @@ -0,0 +1,95 @@ +import json +import os + +import boto3 +import elasticsearch +from elasticsearch import Elasticsearch, RequestsHttpConnection +from elasticsearch.helpers import bulk +from requests_aws4auth import AWS4Auth + + +def build_documents(version, build_folder): + for root, _, files in os.walk(build_folder): + for filename in files: + if not filename.endswith(".fjson"): + continue + abs_path = os.path.join(root, filename) + with open(abs_path, "r") as f: + data = json.load(f) + if not data.get("title"): + continue + title = data["title"] + slug = data["current_page_name"] + ".html" + parent_title = data["parents"][0]["title"] if data["parents"] else "" + html = data["body"] + element = {"version": version, "title": title, "parent_title": parent_title, + "slug": slug, "html": html} + yield element + + +class ElasticManager(object): + + def __init__(self, host, region): + credentials = boto3.Session().get_credentials() + awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, "es") + + es = Elasticsearch( + hosts=[{'host': host, 'port': 443}], + http_auth=awsauth, + use_ssl=True, + verify_certs=True, + connection_class=RequestsHttpConnection + ) + + self.es = es + + def _gendata(self, version, folder): + data = build_documents(version, folder) + for doc in data: + doc["_index"] = "docs" + doc["_type"] = "docs" + yield doc + + def create_index(self): + doc = """ +{ + "settings": { + "analysis": { + "analyzer": { + "htmlStripAnalyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": ["standard","lowercase"], + "char_filter": [ + "html_strip" + ] + } + } + } + }, + "mappings": { + "docs": { + "properties": { + "html": {"type": "text", "analyzer": "htmlStripAnalyzer"}, + "title" : { "type" : "text" }, + "parent_title" : { "type" : "text" }, + "version": {"type": "text"}, + "url" : { "type" : "text" } + } + } + } +} +""" + self.es.indices.create(index="docs", body=doc) + + def remove_index(self): + try: + self.es.indices.delete(index="docs") + except elasticsearch.exceptions.NotFoundError: + pass + + def index(self, version, folder): + bulk(self.es, self._gendata(version, folder)) + + def ping(self): + return self.es.info() diff --git a/_elastic/query_examples/index.txt b/_elastic/query_examples/index.txt new file mode 100644 index 000000000000..deee2fbda22b --- /dev/null +++ b/_elastic/query_examples/index.txt @@ -0,0 +1,34 @@ +curl -X DELETE "localhost:9200/docs" + +curl -X PUT "localhost:9200/docs" -H 'Content-Type: application/json' -d' +{ + "settings": { + "analysis": { + "analyzer": { + "htmlStripAnalyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": ["standard","lowercase"], + "char_filter": [ + "html_strip" + ] + } + } + } + }, + "mappings": { + "docs": { + "properties": { + "html": {"type": "text", "analyzer": "htmlStripAnalyzer"}, + "title" : { "type" : "text" }, + "parent_title" : { "type" : "text" }, + "version": {"type": "text"}, + "url" : { "type" : "text" } + } + } + } +}' + + + + diff --git a/_elastic/query_examples/insert_example.txt b/_elastic/query_examples/insert_example.txt new file mode 100644 index 000000000000..3ce696e5b3af --- /dev/null +++ b/_elastic/query_examples/insert_example.txt @@ -0,0 +1,45 @@ +curl -X POST "localhost:9200/docs/_doc/integrations\/custom" -H 'Content-Type: application/json' -d' +{ + "version": "1.15", + "parent_title" : "Integrations", + "url": "integrations/custom", + "title" : "Custom integrations", + "html" : "trying out Elasticsearch" +} +' + + + +curl -X POST "localhost:9200/docs/_doc/" -H 'Content-Type: application/json' -d' +{ + "version": "1.15", + "parent_title" : "Integrations", + "url": "integrations/custom", + "title" : "Custom integrations", + "html" : "trying out integrations" +} +' + + + +curl -X POST "localhost:9200/docs/_doc/" -H 'Content-Type: application/json' -d' +{ + "version": "1.15", + "parent_title" : "Integrations", + "url": "integrations/custom", + "title" : "Patata", + "html" : "trying out integrations" +} +' + + +curl -X POST "localhost:9200/docs/_doc/" -H 'Content-Type: application/json' -d' +{ + "version": "1.15", + "parent_title" : "Integrations", + "url": "integrations/custom", + "title" : "Other title", + "html" : "trying out integrations patata" +} +' + diff --git a/_elastic/query_examples/query.txt b/_elastic/query_examples/query.txt new file mode 100644 index 000000000000..adb39fd84de2 --- /dev/null +++ b/_elastic/query_examples/query.txt @@ -0,0 +1,38 @@ +curl -X GET "localhost:9200/docs/_search" -H 'Content-Type: application/json' -d' +{ + "from" : 0, "size" : 5, + "query": { + "bool": { + "filter": [ + { "match": { "version": "1.15"}} + ], + "should": [ + { "match": { + "html": { + "query": "patata", + "boost": 1 + } + }}, + { "match": { + "title": { + "query": "patata", + "boost": 4 + } + }} + ] + } + } +} +' + + + +curl -X GET "localhost:9200/docs/_search" -H 'Content-Type: application/json' -d' +{ + "query": { + "match" : { + "version" : "1.15" + } + } +} +' diff --git a/_themes/conan/breadcrumbs.html b/_themes/conan/breadcrumbs.html index 486905c5754a..c25a4cfae147 100644 --- a/_themes/conan/breadcrumbs.html +++ b/_themes/conan/breadcrumbs.html @@ -24,7 +24,7 @@ {% set display_gitlab = True %} {% endif %} -
diff --git a/_themes/conan/layout.html b/_themes/conan/layout.html index ba63dfd3d4ae..92cd54009ad0 100644 --- a/_themes/conan/layout.html +++ b/_themes/conan/layout.html @@ -49,6 +49,9 @@ {% endfor %} + + + {%- block linktags %} {%- if hasdoc('about') %} @@ -148,9 +150,14 @@ {# PAGE CONTENT #} +