Merge branch 'main' into production

.github/workflows/pylint.yml

@@ -21,8 +21,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install pylint
     - name: Analysing the code with pylint
       run: |
-        pylint bookwyrm/ --ignore=migrations --disable=E1101,E1135,E1136,R0903,R0901,R0902,W0707,W0511,W0406,R0401,R0801
+        pylint bookwyrm/
 

.pylintrc

@@ -0,0 +1,9 @@
+[MAIN]
+ignore=migrations
+load-plugins=pylint.extensions.no_self_use
+
+[MESSAGES CONTROL]
+disable=E1101,E1135,E1136,R0903,R0901,R0902,W0707,W0511,W0406,R0401,R0801,C3001,import-error
+
+[FORMAT]
+max-line-length=88

Dockerfile

@@ -6,5 +6,7 @@ RUN mkdir /app /app/static /app/images
 
 WORKDIR /app
 
+RUN apt-get update && apt-get install -y gettext libgettextpo-dev tidy && apt-get clean
+
 COPY requirements.txt /app/
 RUN pip install -r requirements.txt --no-cache-dir

SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Please report security issues to `[email protected]`

bookwyrm/connectors/connector_manager.py

@@ -152,6 +152,15 @@ def load_more_data(connector_id, book_id):
     connector.expand_book_data(book)
 
 
+@app.task(queue="low_priority")
+def create_edition_task(connector_id, work_id, data):
+    """separate task for each of the 10,000 editions of LoTR"""
+    connector_info = models.Connector.objects.get(id=connector_id)
+    connector = load_connector(connector_info)
+    work = models.Work.objects.select_subclasses().get(id=work_id)
+    connector.create_edition_from_data(work, data)
+
+
 def load_connector(connector_info):
     """instantiate the connector class"""
     connector = importlib.import_module(

bookwyrm/connectors/inventaire.py

@@ -5,7 +5,7 @@
 from bookwyrm.book_search import SearchResult
 from .abstract_connector import AbstractConnector, Mapping
 from .abstract_connector import get_data
-from .connector_manager import ConnectorException
+from .connector_manager import ConnectorException, create_edition_task
 
 
 class Connector(AbstractConnector):
@@ -156,12 +156,17 @@ def expand_book_data(self, book):
 
         for edition_uri in edition_options.get("uris"):
             remote_id = self.get_remote_id(edition_uri)
+            create_edition_task.delay(self.connector.id, work.id, remote_id)
+
+    def create_edition_from_data(self, work, edition_data, instance=None):
+        """pass in the url as data and then call the version in abstract connector"""
+        if isinstance(edition_data, str):
             try:
-                data = self.get_book_data(remote_id)
+                edition_data = self.get_book_data(edition_data)
             except ConnectorException:
                 # who, indeed, knows
-                continue
-            self.create_edition_from_data(work, data)
+                return
+        super().create_edition_from_data(work, edition_data, instance=instance)
 
     def get_cover_url(self, cover_blob, *_):
         """format the relative cover url into an absolute one:

bookwyrm/connectors/openlibrary.py

@@ -5,7 +5,7 @@
 from bookwyrm.book_search import SearchResult
 from .abstract_connector import AbstractConnector, Mapping
 from .abstract_connector import get_data, infer_physical_format, unique_physical_format
-from .connector_manager import ConnectorException
+from .connector_manager import ConnectorException, create_edition_task
 from .openlibrary_languages import languages
 
 
@@ -153,19 +153,25 @@ def get_cover_url(self, cover_blob, size="L"):
         return f"{self.covers_url}/b/id/{image_name}"
 
     def parse_search_data(self, data, min_confidence):
-        for search_result in data.get("docs"):
+        for idx, search_result in enumerate(data.get("docs")):
             # build the remote id from the openlibrary key
             key = self.books_url + search_result["key"]
             author = search_result.get("author_name") or ["Unknown"]
             cover_blob = search_result.get("cover_i")
             cover = self.get_cover_url([cover_blob], size="M") if cover_blob else None
+
+            # OL doesn't provide confidence, but it does sort by an internal ranking, so
+            # this confidence value is relative to the list position
+            confidence = 1 / (idx + 1)
+
             yield SearchResult(
                 title=search_result.get("title"),
                 key=key,
                 author=", ".join(author),
                 connector=self,
                 year=search_result.get("first_publish_year"),
                 cover=cover,
+                confidence=confidence,
             )
 
     def parse_isbn_search_data(self, data):
@@ -204,7 +210,7 @@ def expand_book_data(self, book):
             # does this edition have ANY interesting data?
             if ignore_edition(edition_data):
                 continue
-            self.create_edition_from_data(work, edition_data)
+            create_edition_task.delay(self.connector.id, work.id, edition_data)
 
 
 def ignore_edition(edition_data):

bookwyrm/importers/calibre_import.py

@@ -24,5 +24,5 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
     def get_shelf(self, normalized_row):
-        # Calibre export does not indicate which shelf to use. Go with a default one for now
+        # Calibre export does not indicate which shelf to use. Use a default one for now
         return Shelf.TO_READ

bookwyrm/lists_stream.py

@@ -114,12 +114,20 @@ def get_objects_for_store(self, store):
 
 @receiver(signals.post_save, sender=models.List)
 # pylint: disable=unused-argument
-def add_list_on_create(sender, instance, created, *args, **kwargs):
-    """add newly created lists streamsstreams"""
-    if not created:
+def add_list_on_create(sender, instance, created, *args, update_fields=None, **kwargs):
+    """add newly created lists streams"""
+    if created:
+        # when creating new things, gotta wait on the transaction
+        transaction.on_commit(lambda: add_list_on_create_command(instance.id))
         return
-    # when creating new things, gotta wait on the transaction
-    transaction.on_commit(lambda: add_list_on_create_command(instance.id))
+
+    # if update_fields was specified, we can check if privacy was updated, but if
+    # it wasn't specified (ie, by an activitypub update), there's no way to know
+    if update_fields and "privacy" not in update_fields:
+        return
+
+    # the privacy may have changed, so we need to re-do the whole thing
+    remove_list_task.delay(instance.id, re_add=True)
 
 
 @receiver(signals.post_delete, sender=models.List)
@@ -217,7 +225,7 @@ def populate_lists_task(user_id):
 
 
 @app.task(queue=MEDIUM)
-def remove_list_task(list_id):
+def remove_list_task(list_id, re_add=False):
     """remove a list from any stream it might be in"""
     stores = models.User.objects.filter(local=True, is_active=True).values_list(
         "id", flat=True
@@ -227,6 +235,9 @@ def remove_list_task(list_id):
     stores = [ListsStream().stream_id(idx) for idx in stores]
     ListsStream().remove_object_from_related_stores(list_id, stores=stores)
 
+    if re_add:
+        add_list_task.delay(list_id)
+
 
 @app.task(queue=HIGH)
 def add_list_task(list_id):

bookwyrm/management/commands/generate_preview_images.py

@@ -56,12 +56,17 @@ def handle(self, *args, **options):
             self.stdout.write(" OK 🖼")
 
             # Books
-            books = models.Book.objects.select_subclasses().filter()
+            book_ids = (
+                models.Book.objects.select_subclasses()
+                .filter()
+                .values_list("id", flat=True)
+            )
+
             self.stdout.write(
-                "   → Book preview images ({}): ".format(len(books)), ending=""
+                "   → Book preview images ({}): ".format(len(book_ids)), ending=""
             )
-            for book in books:
-                preview_images.generate_edition_preview_image_task.delay(book.id)
+            for book_id in book_ids:
+                preview_images.generate_edition_preview_image_task.delay(book_id)
                 self.stdout.write(".", ending="")
             self.stdout.write(" OK 🖼")
 

bookwyrm/models/fields.py

@@ -16,7 +16,7 @@
 
 from bookwyrm import activitypub
 from bookwyrm.connectors import get_image
-from bookwyrm.sanitize_html import InputHtmlParser
+from bookwyrm.utils.sanitizer import clean
 from bookwyrm.settings import MEDIA_FULL_URL
 
 
@@ -497,9 +497,7 @@ class HtmlField(ActivitypubFieldMixin, models.TextField):
     def field_from_activity(self, value):
         if not value or value == MISSING:
             return None
-        sanitizer = InputHtmlParser()
-        sanitizer.feed(value)
-        return sanitizer.get_output()
+        return clean(value)
 
 
 class ArrayField(ActivitypubFieldMixin, DjangoArrayField):

bookwyrm/models/list.py

@@ -129,7 +129,7 @@ def save(self, *args, **kwargs):
         """on save, update embed_key and avoid clash with existing code"""
         if not self.embed_key:
             self.embed_key = uuid.uuid4()
-        return super().save(*args, **kwargs)
+        super().save(*args, **kwargs)
 
 
 class ListItem(CollectionItemMixin, BookWyrmModel):
@@ -156,7 +156,7 @@ def save(self, *args, **kwargs):
         super().save(*args, **kwargs)
         # tick the updated date on the parent list
         self.book_list.updated_date = timezone.now()
-        self.book_list.save(broadcast=False)
+        self.book_list.save(broadcast=False, update_fields=["updated_date"])
 
         list_owner = self.book_list.user
         model = apps.get_model("bookwyrm.Notification", require_ready=True)

bookwyrm/models/relationship.py

@@ -218,7 +218,7 @@ def clear_cache(user_subject, user_object):
     """clear relationship cache"""
     cache.delete_many(
         [
-            f"relationship-{user_subject.id}-{user_object.id}",
-            f"relationship-{user_object.id}-{user_subject.id}",
+            f"cached-relationship-{user_subject.id}-{user_object.id}",
+            f"cached-relationship-{user_object.id}-{user_subject.id}",
         ]
     )

bookwyrm/models/status.py

@@ -303,10 +303,17 @@ class Comment(BookStatus):
     @property
     def pure_content(self):
         """indicate the book in question for mastodon (or w/e) users"""
-        return (
-            f'{self.content}<p>(comment on <a href="{self.book.remote_id}">'
-            f'"{self.book.title}"</a>)</p>'
-        )
+        if self.progress_mode == "PG" and self.progress and (self.progress > 0):
+            return_value = (
+                f'{self.content}<p>(comment on <a href="{self.book.remote_id}">'
+                f'"{self.book.title}"</a>, page {self.progress})</p>'
+            )
+        else:
+            return_value = (
+                f'{self.content}<p>(comment on <a href="{self.book.remote_id}">'
+                f'"{self.book.title}"</a>)</p>'
+            )
+        return return_value
 
     activity_serializer = activitypub.Comment
 
@@ -332,10 +339,17 @@ def pure_content(self):
         """indicate the book in question for mastodon (or w/e) users"""
         quote = re.sub(r"^<p>", '<p>"', self.quote)
         quote = re.sub(r"</p>$", '"</p>', quote)
-        return (
-            f'{quote} <p>-- <a href="{self.book.remote_id}">'
-            f'"{self.book.title}"</a></p>{self.content}'
-        )
+        if self.position_mode == "PG" and self.position and (self.position > 0):
+            return_value = (
+                f'{quote} <p>-- <a href="{self.book.remote_id}">'
+                f'"{self.book.title}"</a>, page {self.position}</p>{self.content}'
+            )
+        else:
+            return_value = (
+                f'{quote} <p>-- <a href="{self.book.remote_id}">'
+                f'"{self.book.title}"</a></p>{self.content}'
+            )
+        return return_value
 
     activity_serializer = activitypub.Quotation
 

bookwyrm/settings.py

@@ -11,7 +11,7 @@
 env = Env()
 env.read_env()
 DOMAIN = env("DOMAIN")
-VERSION = "0.4.0"
+VERSION = "0.4.1"
 
 RELEASE_API = env(
     "RELEASE_API",

bookwyrm/status.py

@@ -2,15 +2,13 @@
 from django.db import transaction
 
 from bookwyrm import models
-from bookwyrm.sanitize_html import InputHtmlParser
+from bookwyrm.utils import sanitizer
 
 
 def create_generated_note(user, content, mention_books=None, privacy="public"):
     """a note created by the app about user activity"""
     # sanitize input html
-    parser = InputHtmlParser()
-    parser.feed(content)
-    content = parser.get_output()
+    content = sanitizer.clean(content)
 
     with transaction.atomic():
         # create but don't save

bookwyrm/templates/about/layout.html

@@ -50,7 +50,7 @@ <h2 class="menu-label">{% blocktrans with site_name=site.name %}About {{ site_na
         </ul>
     </nav>
 
-    <div class="column">
+    <div class="column is-clipped">
         {% block about_content %}{% endblock %}
     </div>
 </div>