TM: upgrade code to elasticsearch 7.x

The most notable change is how multiple mapping types are not supported
in indices. As suggested by the official docs, we'll create separate
indexes for each locale, and hence avoid clashes. Note how index names
must be lowercase, too.

This change, as well as upgrading to elasticsearch 7.x, requires servers
to rebuild their indices.

Author: julen

docs/ref-settings.md

@@ -258,11 +258,7 @@ ZING_TM_SERVER = {
 
 The TM is automatically updated every time a new translation is submitted.
 
-Optionally, a couple of details can be configured:
-
-* `INDEX_NAME` (_string_) allows defining the name of the index under which the
-  Translation Memory database will be constructed. It defaults to
-  `translations`.
+Optionally, matching similarity can be configured:
 
 * `MIN_SIMILARITY` (_float_) serves as a threshold value to filter out results
   that are potentially too far from the source text. The Levenshtein distance is

pootle/apps/pootle_app/management/commands/update_tmserver.py

@@ -80,9 +80,9 @@ def get_unit_data(self, unit):
         if unit['submitted_on']:
             mtime = int(dateformat.format(unit['submitted_on'], 'U'))
 
+        index_name = unit['store__translation_project__language__code'].lower()
         return {
-            '_index': self.INDEX_NAME,
-            '_type': unit['store__translation_project__language__code'],
+            '_index': index_name,
             '_id': unit['id'],
             'revision': int(unit['revision']),
             'project': unit['store__translation_project__project__fullname'],

pootle/core/search/backends/elasticsearch.py

@@ -28,7 +28,6 @@
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_INDEX_NAME = 'translations'
 DEFAULT_MIN_SIMILARITY = 0.7
 
 
@@ -69,19 +68,17 @@ class ElasticSearchBackend(SearchBackend):
     def __init__(self):
         super(ElasticSearchBackend, self).__init__()
         self._es = self._get_es_server()
-        self._index_name = self._settings.get('INDEX_NAME', DEFAULT_INDEX_NAME)
-        self._create_index_if_missing()
 
     def _get_es_server(self):
         return Elasticsearch([
             {'host': self._settings['HOST'],
              'port': self._settings['PORT']},
         ])
 
-    def _create_index_if_missing(self):
+    def _create_index_if_missing(self, name):
         try:
-            if not self._es.indices.exists(self._index_name):
-                self._es.indices.create(self._index_name)
+            if not self._es.indices.exists(name):
+                self._es.indices.create(name)
         except ElasticsearchException as e:
             self._log_error(e)
 
@@ -105,8 +102,7 @@ def search(self, unit):
         language = unit.store.translation_project.language.code
         es_res = self._es_call(
             "search",
-            index=self._index_name,
-            doc_type=language,
+            index=language,
             body={
                 "query": {
                     "match": {
@@ -163,10 +159,11 @@ def search(self, unit):
         return res
 
     def update(self, language, obj):
+        index_name = language.lower()
+        self._create_index_if_missing(index_name)
         self._es_call(
             "index",
-            index=self._index_name,
-            doc_type=language,
+            index=index_name,
             body=obj,
             id=obj['id']
         )

pootle/settings/90-local.conf.template

@@ -98,8 +98,6 @@ DATABASES = {
 #ZING_TM_SERVER = {
 #    'HOST': 'localhost',
 #    'PORT': 9200,
-#    # Optional index name
-#    'INDEX_NAME': 'translations',
 #}
 
 #