diff --git a/README.md b/README.md index 611e7fd..4006c4c 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ at [https://django-opensearch-dsl.readthedocs.io](https://django-opensearch-dsl. - Opensearch auto mapping from Django models fields. - Complex field type support (`ObjectField`, `NestedField`). - Index fast using `parallel` indexing. +- Zero-downtime index migration for when the document mapping changes. ## Requirements diff --git a/django_opensearch_dsl/documents.py b/django_opensearch_dsl/documents.py index 398a6cb..260f3d3 100644 --- a/django_opensearch_dsl/documents.py +++ b/django_opensearch_dsl/documents.py @@ -2,17 +2,19 @@ import sys import time from collections import deque +from datetime import datetime from functools import partial from typing import Optional, Iterable from django.db import models from django.db.models import QuerySet, Q -from opensearch_dsl import Document as DSLDocument +from opensearch_dsl.document import Document as DSLDocument, IndexMeta as DSLIndexMeta from opensearchpy.helpers import bulk, parallel_bulk from . import fields from .apps import DODConfig from .exceptions import ModelFieldNotMappedError +from .indices import Index from .management.enums import OpensearchAction from .search import Search from .signals import post_index @@ -44,9 +46,21 @@ } -class Document(DSLDocument): +class IndexMeta(DSLIndexMeta): + """A specialized DSL IndexMeta that specializes the Document Index class.""" + + def __new__(mcs, *args, **kwargs): + """Override `_index` with django_opensearch_dsl Index class.""" + new_cls = super().__new__(mcs, *args, **kwargs) + if new_cls._index and new_cls._index._name: # noqa + new_cls._index.__class__ = Index # noqa + return new_cls + + +class Document(DSLDocument, metaclass=IndexMeta): """Allow the definition of Opensearch' index using Django `Model`.""" + VERSION_NAME_SEPARATOR = "--" _prepared_fields = [] def __init__(self, related_instance_to_ignore=None, **kwargs): @@ -56,6 +70,64 @@ def __init__(self, related_instance_to_ignore=None, **kwargs): self._related_instance_to_ignore = related_instance_to_ignore self._prepared_fields = self.init_prepare() + @classmethod + def get_index_name(cls, suffix=None): + """Compute the concrete Index name for the given (or not) suffix.""" + name = cls._index._name # noqa + if suffix: + name += f"{cls.VERSION_NAME_SEPARATOR}{suffix}" + return name + + @classmethod + def get_all_indices(cls, using=None): + """Fetches from OpenSearch all concrete indices for this Document.""" + return [ + Index(name) + for name in sorted( + cls._get_connection(using=using).indices.get(f"{cls._index._name}{cls.VERSION_NAME_SEPARATOR}*").keys() + ) + ] + + @classmethod + def get_active_index(cls, using=None): + """Return the Index that's active for this Document.""" + for index in cls.get_all_indices(using=using): + if index.exists_alias(name=cls._index._name): # noqa + return index + + @classmethod + def migrate(cls, suffix, using=None): + """Sets an alias of the Document Index name to a given concrete Index.""" + index_name = cls.get_index_name(suffix) + + actions_on_aliases = [ + {"add": {"index": index_name, "alias": cls._index._name}}, # noqa + ] + + active_index = cls.get_active_index() + if active_index: + actions_on_aliases.insert( + 0, + {"remove": {"index": active_index._name, "alias": cls._index._name}}, # noqa + ) + + if len(actions_on_aliases) == 1 and cls._index.exists(): + cls._index.delete() + + cls._get_connection(using=using).indices.update_aliases(body={"actions": actions_on_aliases}) + + @classmethod + def init(cls, suffix=None, using=None): + """Init the Index with a named suffix to handle multiple versions. + + Create an alias to the default index name if it doesn't exist. + """ + suffix = suffix or datetime.now().strftime("%Y%m%d%H%M%S%f") + index_name = cls.get_index_name(suffix) + super().init(index=index_name, using=using) + if not cls._index.exists(): + cls.migrate(suffix, using=using) + @classmethod def search(cls, using=None, index=None): """Return a `Search` object parametrized with the index' information.""" @@ -196,18 +268,18 @@ def generate_id(cls, object_instance): """ return object_instance.pk - def _prepare_action(self, object_instance, action): + def _prepare_action(self, object_instance, action, index_name=None): return { "_op_type": action, - "_index": self._index._name, # noqa + "_index": index_name or self._index._name, # noqa "_id": self.generate_id(object_instance), "_source" if action != "update" else "doc": (self.prepare(object_instance) if action != "delete" else None), } - def _get_actions(self, object_list, action): + def _get_actions(self, object_list, action, **kwargs): for object_instance in object_list: if action == "delete" or self.should_index_object(object_instance): - yield self._prepare_action(object_instance, action) + yield self._prepare_action(object_instance, action, **kwargs) def _bulk(self, *args, parallel=False, using=None, **kwargs): """Helper for switching between normal and parallel bulk operation.""" @@ -223,14 +295,18 @@ def should_index_object(self, obj): """ return True - def update(self, thing, action, *args, refresh=None, using=None, **kwargs): # noqa + def update(self, thing, action, *args, index_suffix=None, refresh=None, using=None, **kwargs): # noqa """Update document in OS for a model, iterable of models or queryset.""" if refresh is None: refresh = getattr(self.Index, "auto_refresh", DODConfig.auto_refresh_enabled()) + index_name = self.__class__.get_index_name(index_suffix) if index_suffix else None + if isinstance(thing, models.Model): object_list = [thing] else: object_list = thing - return self._bulk(self._get_actions(object_list, action), *args, refresh=refresh, using=using, **kwargs) + return self._bulk( + self._get_actions(object_list, action, index_name=index_name), *args, refresh=refresh, using=using, **kwargs + ) diff --git a/django_opensearch_dsl/management/commands/opensearch.py b/django_opensearch_dsl/management/commands/opensearch.py index 03e8532..4db83e4 100644 --- a/django_opensearch_dsl/management/commands/opensearch.py +++ b/django_opensearch_dsl/management/commands/opensearch.py @@ -5,6 +5,7 @@ import sys from argparse import ArgumentParser from collections import defaultdict +from datetime import datetime from typing import Any, Callable import opensearchpy @@ -17,6 +18,7 @@ from django_opensearch_dsl.registries import registry from ..enums import OpensearchAction from ..types import parse +from ...documents import Document class Command(BaseCommand): @@ -54,19 +56,40 @@ def __list_index(self, **options): # noqa pragma: no cover indices = registry.get_indices() result = defaultdict(list) for index in indices: - module = index._doc_types[0].__module__.split(".")[-2] # noqa + document = index._doc_types[0] + module = document.__module__.split(".")[-2] # noqa exists = index.exists() checkbox = f"[{'X' if exists else ' '}]" - count = f" ({index.search().count()} documents)" if exists else "" - result[module].append(f"{checkbox} {index._name}{count}") + document_indices = document.get_all_indices() + if document_indices: + details = ". Following indices exist:" + for document_index in document_indices: + is_active = "" + if document_index.exists_alias(name=index._name): + is_active = self.style.SUCCESS("Active") + count = f" ({document_index.search().count()} documents)" + details += f"\n - {document_index._name} {is_active}{count}" + elif exists: + details = f" ({index.search().count()} documents)" + else: + details = "" + result[module].append(f"{checkbox} {index._name}{details}") for app, indices in result.items(): self.stdout.write(self.style.MIGRATE_LABEL(app)) self.stdout.write("\n".join(indices)) - def _manage_index(self, action, indices, force, verbosity, ignore_error, **options): # noqa + def _manage_index(self, action, indices, suffix, force, verbosity, ignore_error, **options): # noqa """Manage the creation and deletion of indices.""" action = OpensearchAction(action) - known = registry.get_indices() + + suffix = suffix or datetime.now().strftime("%Y%m%d%H%M%S%f") + + if action == OpensearchAction.CREATE: + known = registry.get_indices() + else: + known = [] + for document in [i._doc_types[0] for i in registry.get_indices()]: + known.extend(document.get_all_indices()) # Filter indices if indices: @@ -84,9 +107,15 @@ def _manage_index(self, action, indices, force, verbosity, ignore_error, **optio # Display expected action if verbosity or not force: + if not indices: + self.stdout.write("Nothing to do, exiting") + exit(0) self.stdout.write(f"The following indices will be {action.past}:") for index in indices: - self.stdout.write(f"\t- {index._name}.") # noqa + index_name = index._name # noqa + if action == OpensearchAction.CREATE: + index_name += f"{Document.VERSION_NAME_SEPARATOR}{suffix}" + self.stdout.write(f"\t- {index_name}.") # noqa self.stdout.write("") # Ask for confirmation to continue @@ -101,23 +130,21 @@ def _manage_index(self, action, indices, force, verbosity, ignore_error, **optio pp = action.present_participle.title() for index in indices: + index_name = index._name # noqa + if action == OpensearchAction.CREATE: + index_name += f"{Document.VERSION_NAME_SEPARATOR}{suffix}" + if verbosity: self.stdout.write( - f"{pp} index '{index._name}'...\r", + f"{pp} index '{index_name}'...\r", ending="", ) # noqa self.stdout.flush() try: if action == OpensearchAction.CREATE: - index.create() + index._doc_types[0].init(suffix=suffix) elif action == OpensearchAction.DELETE: index.delete() - else: - try: - index.delete() - except opensearchpy.exceptions.NotFoundError: - pass - index.create() except opensearchpy.exceptions.NotFoundError: if verbosity or not ignore_error: self.stderr.write(f"{pp} index '{index._name}'...{self.style.ERROR('Error (not found)')}") # noqa @@ -126,18 +153,28 @@ def _manage_index(self, action, indices, force, verbosity, ignore_error, **optio exit(1) except opensearchpy.exceptions.RequestError: if verbosity or not ignore_error: - self.stderr.write( - f"{pp} index '{index._name}'... {self.style.ERROR('Error (already exists)')}" - ) # noqa + self.stderr.write(f"{pp} index '{index._name}'... {self.style.ERROR('Error')}") # noqa if not ignore_error: self.stderr.write("exiting...") exit(1) else: if verbosity: - self.stdout.write(f"{pp} index '{index._name}'... {self.style.SUCCESS('OK')}") # noqa + self.stdout.write(f"{pp} index '{index_name}'... {self.style.SUCCESS('OK')}") # noqa def _manage_document( - self, action, indices, force, filters, excludes, verbosity, parallel, count, refresh, missing, **options + self, + action, + indices, + index_suffix, + force, + filters, + excludes, + verbosity, + parallel, + count, + refresh, + missing, + **options, ): # noqa """Manage the creation and deletion of indices.""" action = OpensearchAction(action) @@ -185,7 +222,11 @@ def _manage_document( self.stderr.write(f"Error while filtering on '{model}' (from index '{index._name}'):\n{e}'") # noqa exit(1) else: - s += f"\n\t- {qs} {document.django.model.__name__}." + if action == OpensearchAction.MIGRATE: + prefix = "" + else: + prefix = f" {qs}" + s += f"\n\t- {prefix} {document.django.model.__name__}." # Display expected actions if verbosity or not force: @@ -204,26 +245,32 @@ def _manage_document( result = "\n" for index, kwargs in zip(indices, kwargs_list): document = index._doc_types[0]() # noqa - qs = document.get_indexing_queryset(stdout=self.stdout._out, verbose=verbosity, action=action, **kwargs) - success, errors = document.update( - qs, parallel=parallel, refresh=refresh, action=action, raise_on_error=False - ) - - success_str = self.style.SUCCESS(success) if success else success - errors_str = self.style.ERROR(len(errors)) if errors else len(errors) model = document.django.model.__name__ - if verbosity == 1: - result += f"{success_str} {model} successfully {action.past}, {errors_str} errors:\n" - reasons = defaultdict(int) - for e in errors: # Count occurrence of each error - error = e.get(action, {"result": "unknown error"}).get("result", "unknown error") - reasons[error] += 1 - for reasons, total in reasons.items(): - result += f" - {reasons} : {total}\n" + if action == OpensearchAction.MIGRATE: + document.migrate(index_suffix) + if verbosity >= 1: + result += f"{model} successfully migrated to index {document.get_index_name(suffix=index_suffix)}\n" + else: + qs = document.get_indexing_queryset(stdout=self.stdout._out, verbose=verbosity, action=action, **kwargs) + success, errors = document.update( + qs, action, parallel=parallel, index_suffix=index_suffix, refresh=refresh, raise_on_error=False + ) + + success_str = self.style.SUCCESS(success) if success else success + errors_str = self.style.ERROR(len(errors)) if errors else len(errors) - if verbosity > 1: - result += f"{success_str} {model} successfully {action}d, {errors_str} errors:\n {errors}\n" + if verbosity == 1: + result += f"{success_str} {model} successfully {action.past}, {errors_str} errors:\n" + reasons = defaultdict(int) + for e in errors: # Count occurrence of each error + error = e.get(action, {"result": "unknown error"}).get("result", "unknown error") + reasons[error] += 1 + for reasons, total in reasons.items(): + result += f" - {reasons} : {total}\n" + + if verbosity > 1: + result += f"{success_str} {model} successfully {action}d, {errors_str} errors:\n {errors}\n" if verbosity: self.stdout.write(result + "\n") @@ -241,23 +288,28 @@ def add_arguments(self, parser): ) subparser.set_defaults(func=self.__list_index) - # 'manage' subcommand + # 'index' subcommand subparser = subparsers.add_parser( "index", - help="Manage the creation an deletion of indices.", - description="Manage the creation an deletion of indices.", + help="Manage the deletion of indices.", + description="Manage the deletion of indices.", ) subparser.set_defaults(func=self._manage_index) subparser.add_argument( "action", type=str, - help="Whether you want to create, delete or rebuild the indices.", + help="Whether you want to create or delete the indices.", choices=[ OpensearchAction.CREATE.value, OpensearchAction.DELETE.value, - OpensearchAction.REBUILD.value, ], ) + subparser.add_argument( + "--suffix", + type=str, + default=None, + help="A suffix for the index name to create/delete (if you don't provide one, a timestamp will be used for creation).", + ) subparser.add_argument("--force", action="store_true", default=False, help="Do not ask for confirmation.") subparser.add_argument("--ignore-error", action="store_true", default=False, help="Do not stop on error.") subparser.add_argument( @@ -279,11 +331,12 @@ def add_arguments(self, parser): subparser.add_argument( "action", type=str, - help="Whether you want to create, delete or rebuild the indices.", + help="Whether you want to index, delete or update documents in indices. Or migrate from an index to another.", choices=[ OpensearchAction.INDEX.value, OpensearchAction.DELETE.value, OpensearchAction.UPDATE.value, + OpensearchAction.MIGRATE.value, ], ) subparser.add_argument( @@ -316,6 +369,12 @@ def add_arguments(self, parser): ), ) subparser.add_argument("--force", action="store_true", default=False, help="Do not ask for confirmation.") + subparser.add_argument( + "--index-suffix", + type=str, + default=None, + help="The suffix for the index name (if you don't provide one, the current index will be used). Required for `migrate` subcommand.", + ) subparser.add_argument( "-i", "--indices", type=str, nargs="*", help="Only update documents on the given indices." ) diff --git a/django_opensearch_dsl/management/enums.py b/django_opensearch_dsl/management/enums.py index 0c90918..7953d1e 100644 --- a/django_opensearch_dsl/management/enums.py +++ b/django_opensearch_dsl/management/enums.py @@ -7,10 +7,9 @@ class OpensearchAction(str, Enum): INDEX = ("index", "indexing", "indexed") UPDATE = ("update", "updating", "updated") CREATE = ("create", "creating", "created") - REBUILD = ("rebuild", "rebuilding", "rebuilded") + MIGRATE = ("migrate", "migrating", "migrated") LIST = ("list", "listing", "listed") DELETE = ("delete", "deleting", "deleted") - MANAGE = ("manage", "managing", "managed") def __new__(cls, value: str, present_participle: str, past: str): # noqa: D102 obj = str.__new__(cls, value) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8b96ccc..3e3b189 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## X.Y.Z (not released yet) + +* New feature: zero-downtime document migration + ## 0.5.0 (2022-11-19) * `get_indexing_queryset()` now order unordered QuerySet by their PK. diff --git a/docs/management.md b/docs/management.md index 48192e3..8edec47 100644 --- a/docs/management.md +++ b/docs/management.md @@ -44,20 +44,22 @@ Indices and documents can be managed through the `opensearch` management command ### Summary ```text -usage: manage.py opensearch index [-h] [--force] [--ignore-error] - {create,delete,rebuild} [INDEX [INDEX ...]] +usage: manage.py opensearch index [-h] [--suffix SUFFIX] [--force] + [--ignore-error] + {create,delete} [INDEX [INDEX ...]] -Manage the creation and deletion of indices. +Manage the deletion of indices. positional arguments: - {create,delete,rebuild} - Whether you want to create, delete or rebuild the indices. - INDEX Only manage the given indices. + {create,delete} Whether you want to create or delete the indices. + INDEX Only manage the given indices. optional arguments: - -h, --help show this help message and exit - --force Do not ask for confirmation. - --ignore-error Do not stop on error. + -h, --help show this help message and exit + --suffix SUFFIX A suffix for the index name to create/delete (if you don't + provide one, a timestamp will be used for creation). + --force Do not ask for confirmation. + --ignore-error Do not stop on error. ``` ### Description @@ -66,7 +68,6 @@ This command takes a mandatory positional argument: * `create` - Create the indices. * `delete` - Delete the indices. -* `rebuild` - Rebuild (delete then create) the indices. The command can also take any number of optional positional arguments which are the names of the indices that should be created/deleted. If no index is provided, the action is applied to all indices. @@ -83,8 +84,11 @@ Sample output : ```text django_dummy_app -[X] country (0 documents) -[ ] continent +[X] country. Following indices exist: + - country--20220813172923248055 Active (0 documents) + - country--20220813172923212345 (0 documents) +[ ] continent. Following indices exist: + - continent--20220813172923248055 (0 documents) [X] event (2361 documents) ``` @@ -95,41 +99,40 @@ django_dummy_app ```text usage: manage.py opensearch document [-h] [-f [FILTERS [FILTERS ...]]] [-e [EXCLUDES [EXCLUDES ...]]] [--force] + [--index-suffix INDEX_SUFFIX] [-i [INDICES [INDICES ...]]] [-c COUNT] - [-p] [-r] [-m] {index,delete,update} + [-p] [-r] [-m] + {index,delete,update,migrate} Manage the indexation and creation of documents. positional arguments: - {index,delete,update} - Whether you want to create, delete or rebuild the indices. + {index,delete,update,migrate} + Whether you want to index, delete or update documents in indices. Or migrate from an index to another. optional arguments: -h, --help show this help message and exit -f [FILTERS [FILTERS ...]], --filters [FILTERS [FILTERS ...]] - Filter object in the queryset. Argument must be formatted as - '[lookup]=[value]', e.g. 'document_date__gte=2020-05-21. The accepted - value type are: - - 'None' ('[lookup]=') - - 'float' ('[lookup]=1.12') - - 'int' ('[lookup]=23') - - 'datetime.date' ('[lookup]=2020-10-08') - - 'list' ('[lookup]=1,2,3,4') Value between comma ',' can be of any - other accepted value type - - 'str' ('[lookup]=week') - Values that didn't match any type above will be interpreted as a str. - The list of lookup function can be found here: - https://docs.djangoproject.com/en/dev/ref/models/querysets/#field-lookups + Filter object in the queryset. Argument must be formatted as '[lookup]=[value]', e.g. 'document_date__gte=2020-05-21. + The accepted value type are: + - 'None' ('[lookup]=') + - 'float' ('[lookup]=1.12') + - 'int' ('[lookup]=23') + - 'datetime.date' ('[lookup]=2020-10-08') + - 'list' ('[lookup]=1,2,3,4') Value between comma ',' can be of any other accepted value type + - 'str' ('[lookup]=week') Value that didn't match any type above will be interpreted as a str + The list of lookup function can be found here: https://docs.djangoproject.com/en/dev/ref/models/querysets/#field-lookups -e [EXCLUDES [EXCLUDES ...]], --excludes [EXCLUDES [EXCLUDES ...]] - Exclude objects from the queryset. Argument must be formatted as - '[lookup]=[value]', see '--filters' for more information. + Exclude objects from the queryset. Argument must be formatted as '[lookup]=[value]', see '--filters' for more information --force Do not ask for confirmation. + --index-suffix INDEX_SUFFIX + The suffix for the index name (if you don't provide one, the current index will be used). Required for `migrate` subcommand. -i [INDICES [INDICES ...]], --indices [INDICES [INDICES ...]] Only update documents on the given indices. -c COUNT, --count COUNT Update at most COUNT objects (0 to index everything). -p, --parallel Parallelize the communication with Opensearch. - -r, --refresh Make operations performed on the indices immediately available for search. + -r, --refresh Make operations performed on the indices immediatly available for search. -m, --missing When used with 'index' action, only index documents not indexed yet. ``` @@ -140,6 +143,7 @@ This command allows you to index your model into Opensearch. It takes a required * `index` Index the documents, already indexed documents will be reindexed if you do not use the `--missing` option. * `delete` Documents will be deleted from the index. * `update` Update already indexed documents. +* `migrate` Migrate from an index to another (by switching index alias) ***Choosing indices*** diff --git a/tests/tests/management/test_document.py b/tests/tests/management/test_document.py index e325377..f85c8af 100644 --- a/tests/tests/management/test_document.py +++ b/tests/tests/management/test_document.py @@ -3,11 +3,11 @@ import time from django.test import TestCase +from opensearch_dsl.connections import get_connection from django_dummy_app.commands import call_command from django_dummy_app.documents import CountryDocument, ContinentDocument, EventDocument from django_dummy_app.models import Country, Event, Continent -from django_opensearch_dsl.registries import registry class DocumentTestCase(TestCase): @@ -20,9 +20,7 @@ def setUpClass(cls): cls.call_command = functools.partial(call_command, stdout=devnull, stderr=devnull) def setUp(self) -> None: - indices = registry.get_indices() - for i in indices: - i.delete(ignore_unavailable=True) + get_connection().indices.delete("_all") def test_unknown_index(self): with self.assertRaises(SystemExit): @@ -234,3 +232,11 @@ def test_index_not_refresh(self): self.assertEqual(ContinentDocument.search().count(), 3) self.assertEqual(CountryDocument.search().count(), 3) self.assertEqual(EventDocument.search().count(), 3) + + def test_migrate(self): + self.call_command("opensearch", "index", "create", suffix="v1", force=True) + self.call_command("opensearch", "index", "create", suffix="v2", force=True) + self.assertEqual(CountryDocument.get_active_index()._name, "country--v1") + + self.call_command("opensearch", "document", "migrate", index_suffix="v2", force=True) + self.assertEqual(CountryDocument.get_active_index()._name, "country--v2") diff --git a/tests/tests/management/test_index.py b/tests/tests/management/test_index.py index dcf4ab7..4e11a29 100644 --- a/tests/tests/management/test_index.py +++ b/tests/tests/management/test_index.py @@ -2,6 +2,7 @@ import os from django.test import SimpleTestCase +from opensearch_dsl.connections import get_connection from django_dummy_app.commands import call_command from django_dummy_app.documents import CountryDocument, ContinentDocument, EventDocument @@ -16,15 +17,13 @@ def setUpClass(cls): cls.call_command = functools.partial(call_command, stdout=devnull, stderr=devnull) def setUp(self) -> None: - indices = registry.get_indices() - for i in indices: - i.delete(ignore_unavailable=True) + get_connection().indices.delete("_all") def test_index_creation_all(self): indices = registry.get_indices() self.assertFalse(any(map(lambda i: i.exists(), indices))) - self.call_command("opensearch", "index", "create", force=True) + self.call_command("opensearch", "index", "create", suffix="v1", force=True) self.assertTrue(all(map(lambda i: i.exists(), indices))) def test_index_creation_one(self): @@ -60,16 +59,8 @@ def test_index_creation_two(self): self.assertTrue(country_document._index.exists()) self.assertTrue(event_document._index.exists()) - def test_index_creation_error(self): - country_document = CountryDocument() - self.call_command("opensearch", "index", "create", country_document.Index.name, force=True) - - self.call_command("opensearch", "index", "create", country_document.Index.name, force=True, ignore_error=True) - with self.assertRaises(SystemExit): - self.call_command("opensearch", "index", "create", country_document.Index.name, force=True) - def test_index_deletion_all(self): - self.call_command("opensearch", "index", "create", force=True) + self.call_command("opensearch", "index", "create", suffix="v1", force=True) indices = registry.get_indices() self.assertTrue(all(map(lambda i: i.exists(), indices))) @@ -77,7 +68,7 @@ def test_index_deletion_all(self): self.assertFalse(any(map(lambda i: i.exists(), indices))) def test_index_deletion_one(self): - self.call_command("opensearch", "index", "create", force=True) + self.call_command("opensearch", "index", "create", suffix="v1", force=True) continent_document = ContinentDocument() country_document = CountryDocument() event_document = EventDocument() @@ -85,13 +76,13 @@ def test_index_deletion_one(self): self.assertTrue(continent_document._index.exists()) self.assertTrue(country_document._index.exists()) self.assertTrue(event_document._index.exists()) - self.call_command("opensearch", "index", "delete", country_document.Index.name, force=True) + self.call_command("opensearch", "index", "delete", "country--v1", force=True) self.assertTrue(continent_document._index.exists()) self.assertFalse(country_document._index.exists()) self.assertTrue(event_document._index.exists()) def test_index_deletion_two(self): - self.call_command("opensearch", "index", "create", force=True) + self.call_command("opensearch", "index", "create", suffix="v1", force=True) continent_document = ContinentDocument() country_document = CountryDocument() event_document = EventDocument() @@ -103,8 +94,8 @@ def test_index_deletion_two(self): "opensearch", "index", "delete", - country_document.Index.name, - event_document.Index.name, + "country--v1", + "event--v1", force=True, ) self.assertTrue(continent_document._index.exists()) @@ -113,38 +104,11 @@ def test_index_deletion_two(self): def test_index_deletion_error(self): country_document = CountryDocument() + country_document.init(suffix="v1") - self.call_command("opensearch", "index", "delete", country_document.Index.name, force=True, ignore_error=True) + self.call_command("opensearch", "index", "delete", "country--v1", force=True, ignore_error=True) with self.assertRaises(SystemExit): - self.call_command("opensearch", "index", "delete", country_document.Index.name, force=True) - - def test_index_rebuild_two(self): - continent_document = ContinentDocument() - country_document = CountryDocument() - event_document = EventDocument() - self.call_command( - "opensearch", - "index", - "create", - continent_document.Index.name, - event_document.Index.name, - force=True, - ) - - self.assertTrue(continent_document._index.exists()) - self.assertFalse(country_document._index.exists()) - self.assertTrue(event_document._index.exists()) - self.call_command( - "opensearch", - "index", - "rebuild", - country_document.Index.name, - event_document.Index.name, - force=True, - ) - self.assertTrue(continent_document._index.exists()) - self.assertTrue(country_document._index.exists()) - self.assertTrue(event_document._index.exists()) + self.call_command("opensearch", "index", "delete", "country--v1", force=True) def test_unknown_index(self): with self.assertRaises(SystemExit): diff --git a/tests/tests/test_documents.py b/tests/tests/test_documents.py index 19ab181..01d5ed7 100644 --- a/tests/tests/test_documents.py +++ b/tests/tests/test_documents.py @@ -77,6 +77,10 @@ class Index: class DocumentTestCase(TestCase): fixtures = ["tests/django_dummy_app/geography_data.json"] + def setUp(self) -> None: + for index in CarDocument.get_all_indices(): + index.delete() + def test_model_class_added(self): self.assertEqual(CarDocument.django.model, Car) @@ -94,6 +98,57 @@ class Index: self.assertFalse(CarDocument2.Index.auto_refresh) + def test_init(self): + # GIVEN no existing index for the Document + self.assertFalse(CarDocument._index.exists()) + + # WHEN init() + CarDocument.init() + + # THEN an index has been created with an alias to the abstract Document index name + self.assertTrue(CarDocument._index.exists()) + self.assertEqual(len(CarDocument._index.get_alias().keys()), 1) + + def test_init_with_suffix(self): + # GIVEN no existing index for the Document + self.assertFalse(CarDocument._index.exists()) + + # WHEN init() with explicit suffix + CarDocument.init(suffix="v1") + + # THEN an index has been created with the given name + # and an alias to the abstract Document index name + self.assertTrue(CarDocument._index.exists()) + self.assertEqual(set(CarDocument._index.get_alias().keys()), {"car_index--v1"}) + + def test_get_all_indices(self): + # GIVEN no existing index for the Document + self.assertFalse(CarDocument._index.exists()) + # First ensure get_all_indices() is empty + self.assertEqual(CarDocument.get_all_indices(), []) + + # WHEN creating 2 named indices + CarDocument.init(suffix="v1") + CarDocument.init(suffix="v2") + + # THEN 2 indices have been created and get_all_indices() shows them + self.assertTrue(CarDocument._index.exists()) + self.assertEqual([i._name for i in CarDocument.get_all_indices()], ["car_index--v1", "car_index--v2"]) + + def test_migrate(self): + # GIVEN 2 existing indices for the Document + CarDocument.init(suffix="v1") + CarDocument.init(suffix="v2") + self.assertEqual(len(CarDocument.get_all_indices()), 2) + # First ensure that the alias points to the first version + self.assertEqual(set(CarDocument._index.get_alias().keys()), {"car_index--v1"}) + + # WHEN migrating to v2 + CarDocument.migrate("v2") + + # THEN the alias is now pointing to v2 + self.assertEqual(set(CarDocument._index.get_alias().keys()), {"car_index--v2"}) + def test_queryset_pagination_added(self): @registry.register_document class CarDocument2(Document): diff --git a/tests/tests/test_search.py b/tests/tests/test_search.py index 20f519a..b395237 100644 --- a/tests/tests/test_search.py +++ b/tests/tests/test_search.py @@ -1,21 +1,19 @@ from django.test import TestCase from opensearch_dsl import Q +from opensearch_dsl.connections import get_connection from django_dummy_app.documents import CountryDocument, ContinentDocument from django_dummy_app.models import Country, Continent -from django_opensearch_dsl.registries import registry class DocumentTestCase(TestCase): fixtures = ["tests/django_dummy_app/geography_data.json"] def setUp(self) -> None: - indices = registry.get_indices() - for i in indices: - i.delete(ignore_unavailable=True) + get_connection().indices.delete("_all") def test_search_country(self): - CountryDocument._index.create() + CountryDocument.init() CountryDocument().update(CountryDocument().get_indexing_queryset(), "index", refresh=True) self.assertEqual( @@ -24,7 +22,7 @@ def test_search_country(self): ) def test_search_country_cache(self): - CountryDocument._index.create() + CountryDocument.init() CountryDocument().update(CountryDocument().get_indexing_queryset(), "index", refresh=True) search = CountryDocument.search().query("term", **{"continent.name": "Europe"}).extra(size=300) @@ -34,7 +32,7 @@ def test_search_country_cache(self): ) def test_search_country_keep_order(self): - CountryDocument._index.create() + CountryDocument.init() CountryDocument().update(CountryDocument().get_indexing_queryset(), "index", refresh=True) search = CountryDocument.search().query("term", **{"continent.name": "Europe"}).extra(size=300) @@ -43,7 +41,7 @@ def test_search_country_keep_order(self): ) def test_search_country_refresh_default_to_document(self): - CountryDocument._index.create() + CountryDocument.init() CountryDocument().update(CountryDocument().get_indexing_queryset(), "index", refresh=True) self.assertEqual( @@ -52,7 +50,7 @@ def test_search_country_refresh_default_to_document(self): ) def test_search_country_refresh_default_to_settings(self): - ContinentDocument._index.create() + ContinentDocument.init() ContinentDocument().update(ContinentDocument().get_indexing_queryset(), "index", refresh=True) search = ContinentDocument.search().query( @@ -61,7 +59,7 @@ def test_search_country_refresh_default_to_settings(self): self.assertEqual(set(search.to_queryset()), set(Continent.objects.filter(countries__name="France"))) def test_update_instance(self): - ContinentDocument._index.create() + ContinentDocument.init() ContinentDocument().update(Continent.objects.get(countries__name="France"), "index", refresh=True) search = ContinentDocument.search().query(