From 661c7ce91a084064eb1c41004030e0f64fc1d1c8 Mon Sep 17 00:00:00 2001 From: Marvin Ahlgrimm Date: Wed, 18 Sep 2024 06:49:38 +0200 Subject: [PATCH] Add option to automatically generate slug from another field (#258) * Allow automatically generating a slug from another field * Fix test description * Add encoding param to MySQL/MariDB config * Add testcase * Fix findings * Remove callback doc * Fix base validation * Fix findings * Fix sluggable tests * Allow unicode * Update fields.md * Update base.cr --- .../models-and-databases/reference/fields.md | 31 +++++++ spec/marten/core/sluggable_spec.cr | 65 ++++++++++++++ spec/marten/db/field/slug_spec.cr | 90 +++++++++++++++++++ spec/marten/db/field/slug_spec/app.cr | 5 ++ .../db/field/slug_spec/models/article.cr | 7 ++ .../models/article_invalid_slug_field.cr | 7 ++ .../slug_spec/models/article_long_slug.cr | 7 ++ spec/test_project.cr | 2 + src/marten/core/sluggable.cr | 23 +++++ src/marten/db/field/base.cr | 16 ++-- src/marten/db/field/slug.cr | 28 +++++- 11 files changed, 271 insertions(+), 10 deletions(-) create mode 100644 spec/marten/core/sluggable_spec.cr create mode 100644 spec/marten/db/field/slug_spec/app.cr create mode 100644 spec/marten/db/field/slug_spec/models/article.cr create mode 100644 spec/marten/db/field/slug_spec/models/article_invalid_slug_field.cr create mode 100644 spec/marten/db/field/slug_spec/models/article_long_slug.cr create mode 100644 src/marten/core/sluggable.cr diff --git a/docs/docs/models-and-databases/reference/fields.md b/docs/docs/models-and-databases/reference/fields.md index ffa8e101d..ecacb4396 100644 --- a/docs/docs/models-and-databases/reference/fields.md +++ b/docs/docs/models-and-databases/reference/fields.md @@ -244,6 +244,37 @@ A `slug` field allows to persist _valid_ slug values (ie. strings that can only The `max_size` argument is optional and defaults to 50 characters. It allows to specify the maximum size of the persisted email addresses. This maximum size is used for the corresponding column definition and when it comes to validate field values. +#### `slugify` + +The `slugify` argument allows specifying the field from which the slug should be generated. This is useful when you want the slug to be automatically derived from another field. + +```crystal +class Article < Marten::Model + field :title, :string + field :slug, :slug, slugify: :title +end + +article = Article.create!(title: "My Article") + +article.slug # => "my-article" +``` + +When an Article object is saved, the slug field will automatically generate a slug based on the title field if no custom slug is provided. + +:::warning +The slugification functionality also transforms Unicode characters and symbols. When filtering a model by the query parameter, it may be necessary to decode the slug parameter first, because although the browser may show you the unicode characters, it will send the encoded characters in the HTTP request: + +```crystal +class ArticleDetailsHandler < Marten::Handler + def get + article = Article.get(slug: URI.decode(params[:slug].to_s)) + + # … + end +end +``` +::: + :::info As slug fields are usually used to query records, they are indexed by default. You can use the [`index`](#index) option (`index: false`) to disable auto-indexing. ::: diff --git a/spec/marten/core/sluggable_spec.cr b/spec/marten/core/sluggable_spec.cr new file mode 100644 index 000000000..8c8b03a3a --- /dev/null +++ b/spec/marten/core/sluggable_spec.cr @@ -0,0 +1,65 @@ +require "./spec_helper" + +describe Marten::Core::Sluggable do + describe "#generate_slug" do + value = "Test Title 12354543435" + max_size = 20 + g_slug = Marten::Core::SluggableSpec::SlugGenerator.new.generate_slug(value, max_size) + + it "removes non-alphanumeric characters" do + value_with_special_chars = "Test@Title#123!" + slug = Marten::Core::SluggableSpec::SlugGenerator.new.generate_slug(value_with_special_chars, max_size) + slug.should eq("testtitle123") + end + + it "handles emojis" do + value_with_emojis = "🚀 TRAVEL & PLACES" + slug = Marten::Core::SluggableSpec::SlugGenerator.new.generate_slug(value_with_emojis, max_size) + slug.should eq("🚀-travel-places") + end + it "converts the string to lowercase" do + g_slug.should eq("test-title-123545434") + end + + it "replaces whitespace and hyphens with a single hyphen" do + value_with_spaces_and_hyphens = "Test - Title 123" + slug = Marten::Core::SluggableSpec::SlugGenerator.new.generate_slug(value_with_spaces_and_hyphens, max_size) + slug.should eq("test-title-123") + end + + it "strips leading and trailing hyphens and underscores" do + value_with_hyphens = "-Test Title 123-" + slug = Marten::Core::SluggableSpec::SlugGenerator.new.generate_slug(value_with_hyphens, max_size) + slug.should eq("test-title-123") + end + + it "removes non-ASCII characters" do + value_with_non_ascii = "Test Títle 123" + slug = Marten::Core::SluggableSpec::SlugGenerator.new.generate_slug(value_with_non_ascii, max_size) + slug.should eq("test-títle-123") + end + + it "limits the slug length to max_size" do + g_slug.size.should eq(max_size) + end + + it "does not exceed max_size even with long input" do + long_value = "This is a very long title that should be truncated" + slug = Marten::Core::SluggableSpec::SlugGenerator.new.generate_slug(long_value, max_size) + slug.size.should be <= max_size + end + + it "does not truncate the slug when max_size is large enough" do + long_value = "This is a very long title that should not be truncated" + slug = Marten::Core::SluggableSpec::SlugGenerator.new.generate_slug(long_value, 100) + + slug.should eq("this-is-a-very-long-title-that-should-not-be-truncated") + end + end +end + +module Marten::Core::SluggableSpec + class SlugGenerator + include Marten::Core::Sluggable + end +end diff --git a/spec/marten/db/field/slug_spec.cr b/spec/marten/db/field/slug_spec.cr index 1d060839a..844c9355d 100644 --- a/spec/marten/db/field/slug_spec.cr +++ b/spec/marten/db/field/slug_spec.cr @@ -1,4 +1,5 @@ require "./spec_helper" +require "./slug_spec/**" describe Marten::DB::Field::Slug do describe "#max_size" do @@ -23,6 +24,95 @@ describe Marten::DB::Field::Slug do end end + describe "slugify" do + with_installed_apps Marten::DB::Field::SlugSpec::App + + it "automatically generates a slug from the title field and assigns it to the slug field if no slug is given" do + article = Marten::DB::Field::SlugSpec::Article.new(title: "My First Article") + + article.save + + article.slug.not_nil!.should eq("my-first-article") + end + + it "raises an error if an invalid field is targetted" do + article = Marten::DB::Field::SlugSpec::ArticleInvalidSlugField.new(title: "My First Article") + + expect_raises(Marten::DB::Errors::UnknownField) do + article.save + end + end + + it "automatically generating a slug does not raise an error" do + article = Marten::DB::Field::SlugSpec::Article.new(title: "My First Article") + + article.save + + article.errors.size.should eq(0) + end + + it "automatically generating a slug with a blank slug value does not raise an error" do + article = Marten::DB::Field::SlugSpec::Article.new(title: "My First Article", slug: "") + + article.save + + article.errors.size.should eq(0) + end + + it "truncates the slug to fit the max size of 50 and appends a random suffix" do + article = Marten::DB::Field::SlugSpec::Article.new( + title: "My First Article: Exploring the Intricacies of Quantum Mechanics" + ) + + article.save + + article.slug.not_nil!.includes?("quantum").should_not be_true + article.slug.not_nil!.size.should eq(50) + end + + it "does not truncate the slug if max size is greater than the string length" do + article = Marten::DB::Field::SlugSpec::ArticleLongSlug.new( + title: "My First Article: Exploring the Intricacies of Quantum Mechanics" + ) + + article.save + + article.slug.not_nil!.includes?("quantum").should be_true + end + + it "removes non-ASCII characters and slugifies the title" do + article = Marten::DB::Field::SlugSpec::Article.new(title: "Überraschungsmoment") + + article.save + + article.slug.not_nil!.should eq("überraschungsmoment") + end + + it "removes emoji and special characters and slugifies the title" do + article = Marten::DB::Field::SlugSpec::Article.new(title: "🚀 TRAVEL & PLACES") + + article.save + + article.slug.not_nil!.should eq("🚀-travel-places") + end + + it "trims leading and trailing whitespace and slugifies the title" do + article = Marten::DB::Field::SlugSpec::Article.new(title: " Test Article ") + + article.save + + article.slug.not_nil!.should eq("test-article") + end + + it "retains a custom slug if provided" do + article = Marten::DB::Field::SlugSpec::Article.new(title: "My First Article", slug: "custom-slug") + + article.save + + article.slug.not_nil!.should eq("custom-slug") + end + end + describe "#validate" do it "does not add an error to the record if the string contains a valid slug" do obj = Tag.new(name: nil) diff --git a/spec/marten/db/field/slug_spec/app.cr b/spec/marten/db/field/slug_spec/app.cr new file mode 100644 index 000000000..5170117eb --- /dev/null +++ b/spec/marten/db/field/slug_spec/app.cr @@ -0,0 +1,5 @@ +require "./models/**" + +class Marten::DB::Field::SlugSpec::App < Marten::App + label :marten_db_field_slug_spec +end diff --git a/spec/marten/db/field/slug_spec/models/article.cr b/spec/marten/db/field/slug_spec/models/article.cr new file mode 100644 index 000000000..9509f74e4 --- /dev/null +++ b/spec/marten/db/field/slug_spec/models/article.cr @@ -0,0 +1,7 @@ +module Marten::DB::Field::SlugSpec + class Article < Marten::Model + field :id, :big_int, primary_key: true, auto: true + field :title, :string, max_size: 255 + field :slug, :slug, slugify: :title + end +end diff --git a/spec/marten/db/field/slug_spec/models/article_invalid_slug_field.cr b/spec/marten/db/field/slug_spec/models/article_invalid_slug_field.cr new file mode 100644 index 000000000..38fd9b30f --- /dev/null +++ b/spec/marten/db/field/slug_spec/models/article_invalid_slug_field.cr @@ -0,0 +1,7 @@ +module Marten::DB::Field::SlugSpec + class ArticleInvalidSlugField < Marten::Model + field :id, :big_int, primary_key: true, auto: true + field :title, :string, max_size: 255 + field :slug, :slug, slugify: :invalid, max_size: 100 + end +end diff --git a/spec/marten/db/field/slug_spec/models/article_long_slug.cr b/spec/marten/db/field/slug_spec/models/article_long_slug.cr new file mode 100644 index 000000000..0ec2c145d --- /dev/null +++ b/spec/marten/db/field/slug_spec/models/article_long_slug.cr @@ -0,0 +1,7 @@ +module Marten::DB::Field::SlugSpec + class ArticleLongSlug < Marten::Model + field :id, :big_int, primary_key: true, auto: true + field :title, :string, max_size: 255 + field :slug, :slug, slugify: :title, max_size: 100 + end +end diff --git a/spec/test_project.cr b/spec/test_project.cr index a107295b3..2567a8424 100644 --- a/spec/test_project.cr +++ b/spec/test_project.cr @@ -23,6 +23,7 @@ Marten.configure :test do |config| db.user = env_settings["MARIADB_DB_USER"].as(String) db.password = env_settings["MARIADB_DB_PASSWORD"].as(String) db.host = env_settings["MARIADB_DB_HOST"].as(String) + db.options = {"encoding" => "utf8mb4"} end config.database :other do |db| @@ -41,6 +42,7 @@ Marten.configure :test do |config| db.user = env_settings["MYSQL_DB_USER"].as(String) db.password = env_settings["MYSQL_DB_PASSWORD"].as(String) db.host = env_settings["MYSQL_DB_HOST"].as(String) + db.options = {"encoding" => "utf8mb4"} end config.database :other do |db| diff --git a/src/marten/core/sluggable.cr b/src/marten/core/sluggable.cr new file mode 100644 index 000000000..6fffa2ada --- /dev/null +++ b/src/marten/core/sluggable.cr @@ -0,0 +1,23 @@ +module Marten + module Core + # The Sluggable module provides functionality for generating URL-friendly slugs from strings. + module Sluggable + private NON_ALPHANUMERIC_RE = /[^\p{L}\p{N}\p{So}\s-]/ + private WHITESPACE_HYPHEN_RE = /[-\s]+/ + + # Generates a slug from the given value, ensuring the resulting slug does not exceed the specified max_size. + # + # The slug is created by: + # 1. Removing non-alphanumeric characters (except for Unicode letters, numbers, symbols, whitespace, and hyphens). + # 2. Converting the string to lowercase. + # 3. Replacing sequences of whitespace and hyphens with a single hyphen. + # 4. Stripping trailing hyphens and underscores from the slug. + # 5. Truncating the slug to the specified max_size. + def generate_slug(value, max_size) + slug = value.gsub(NON_ALPHANUMERIC_RE, "").downcase + slug = slug.gsub(WHITESPACE_HYPHEN_RE, "-") + slug[...(max_size)].strip("-_") + end + end + end +end diff --git a/src/marten/db/field/base.cr b/src/marten/db/field/base.cr index cc9268dd2..6866fb018 100644 --- a/src/marten/db/field/base.cr +++ b/src/marten/db/field/base.cr @@ -86,11 +86,7 @@ module Marten def perform_validation(record : Model) value = record.get_field_value(id) - if value.nil? && !@null - record.errors.add(id, null_error_message(record), type: :null) - elsif empty_value?(value) && !@blank - record.errors.add(id, blank_error_message(record), type: :blank) - end + validate_presence(record, value) validate(record, value) end @@ -131,7 +127,7 @@ module Marten raise NotImplementedError.new("#relation_name must be implemented by subclasses if necessary") end - # Returns `true` if the if the value is considered truthy by the field. + # Returns `true` if the value is considered truthy by the field. def truthy_value?(value) !(value == false || value == 0 || value.nil?) end @@ -147,6 +143,14 @@ module Marten def validate(record, value) end + protected def validate_presence(record : Model, value) + if value.nil? && !@null + record.errors.add(id, null_error_message(record), type: :null) + elsif empty_value?(value) && !@blank + record.errors.add(id, blank_error_message(record), type: :blank) + end + end + # :nodoc: macro check_definition(field_id, kwargs) end diff --git a/src/marten/db/field/slug.cr b/src/marten/db/field/slug.cr index 65d12a694..0c47d2673 100644 --- a/src/marten/db/field/slug.cr +++ b/src/marten/db/field/slug.cr @@ -3,8 +3,10 @@ require "./string" module Marten module DB module Field - # Represents an slug field. + # Represents a slug field. class Slug < String + include Core::Sluggable + def initialize( @id : ::String, @max_size : ::Int32 = 50, @@ -14,10 +16,22 @@ module Marten @null = false, @unique = false, @index = true, - @db_column = nil + @db_column = nil, + @slugify : Symbol? = nil ) end + macro check_definition(field_id, kwargs) + # No-op max_size automatic checks... + end + + def prepare_save(record, new_record = false) + if slugify?(record.get_field_value(id)) + slug = generate_slug(record.get_field_value(slugify.not_nil!).to_s, max_size) + record.set_field_value(id, slug) + end + end + def validate(record, value) return if !value.is_a?(::String) @@ -29,8 +43,14 @@ module Marten end end - macro check_definition(field_id, kwargs) - # No-op max_size automatic checks... + protected def validate_presence(record : Model, value) + super if slugify.nil? + end + + private getter slugify + + private def slugify?(value) + slugify && (value.nil? || (value.is_a?(::String) && value.blank?)) end end end