feat: PG full-text search columns (PPT-2644, 1/2)

migration/db/migrations/20260806100500000_add_search_vector_columns.sql

@@ -0,0 +1,275 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+-- PPT-2644: PostgreSQL full-text search columns replacing Elasticsearch.
+-- Every searchable table gets a generated, stored tsvector maintained by
+-- PostgreSQL itself and a GIN index for fast matching. The 'simple'
+-- configuration is used on both write and query side (see guests.tsv_search
+-- precedent): entity names, identifiers and emails must not be stemmed.
+-- Secrets and encrypted content (password digests, tokens, client secrets,
+-- IdP certificates, settings_string) are deliberately excluded from vectors.
+
+-- array_to_string is only STABLE, so generated columns can't call it
+-- directly; for text[] input it is in fact immutable, hence this wrapper.
+-- +micrate StatementBegin
+CREATE OR REPLACE FUNCTION placeos_fts_join(arr text[]) RETURNS text
+LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
+  SELECT COALESCE(array_to_string(arr, ' '), '')
+$$;
+-- +micrate StatementEnd
+
+-- Emails are additionally split on [@._] so partial-token search works
+-- (e.g. "reeves" or "place" matches "cam.reeves@place.tech"), while the
+-- raw address is kept for whole-address matching.
+-- +micrate StatementBegin
+CREATE OR REPLACE FUNCTION placeos_fts_email(addr text) RETURNS text
+LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
+  SELECT regexp_replace(COALESCE(addr, ''), '[@._]', ' ', 'g') || ' ' || COALESCE(addr, '')
+$$;
+-- +micrate StatementEnd
+
+ALTER TABLE "sys" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(display_name, '') || ' ' || COALESCE(code, '') || ' ' ||
+    COALESCE(type, '') || ' ' || COALESCE(description, '') || ' ' ||
+    placeos_fts_join(features) || ' ' || placeos_fts_email(email) || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_sys_search_vector ON "sys" USING GIN (search_vector);
+
+ALTER TABLE "mod" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(custom_name, '') || ' ' || COALESCE(name, '') || ' ' || COALESCE(ip, '') || ' ' ||
+    COALESCE(uri, '') || ' ' || COALESCE(notes, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_mod_search_vector ON "mod" USING GIN (search_vector);
+
+ALTER TABLE "driver" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(module_name, '') || ' ' || COALESCE(file_name, '') || ' ' ||
+    COALESCE(description, '') || ' ' || COALESCE(default_uri, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_driver_search_vector ON "driver" USING GIN (search_vector);
+
+ALTER TABLE "zone" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(display_name, '') || ' ' || COALESCE(code, '') || ' ' ||
+    COALESCE(type, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(location, '') || ' ' ||
+    placeos_fts_join(tags) || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_zone_search_vector ON "zone" USING GIN (search_vector);
+
+ALTER TABLE "user" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(nickname, '') || ' ' || placeos_fts_email(email) || ' ' ||
+    COALESCE(login_name, '') || ' ' || COALESCE(staff_id, '') || ' ' ||
+    COALESCE(first_name, '') || ' ' || COALESCE(last_name, '') || ' ' ||
+    COALESCE(department, '') || ' ' || COALESCE(building, '') || ' ' ||
+    COALESCE(phone, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_user_search_vector ON "user" USING GIN (search_vector);
+
+ALTER TABLE "repo" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(folder_name, '') || ' ' || COALESCE(uri, '') || ' ' ||
+    COALESCE(description, '') || ' ' || COALESCE(branch, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_repo_search_vector ON "repo" USING GIN (search_vector);
+
+ALTER TABLE "trigger" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_trigger_search_vector ON "trigger" USING GIN (search_vector);
+
+ALTER TABLE "authority" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(domain, '') || ' ' || COALESCE(description, '') || ' ' ||
+    COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_authority_search_vector ON "authority" USING GIN (search_vector);
+
+ALTER TABLE "ldap_strat" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple', COALESCE(name, '') || ' ' || COALESCE(id::text, ''))
+) STORED;
+CREATE INDEX idx_ldap_strat_search_vector ON "ldap_strat" USING GIN (search_vector);
+
+ALTER TABLE "oauth_strat" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple', COALESCE(name, '') || ' ' || COALESCE(id::text, ''))
+) STORED;
+CREATE INDEX idx_oauth_strat_search_vector ON "oauth_strat" USING GIN (search_vector);
+
+ALTER TABLE "adfs_strat" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple', COALESCE(name, '') || ' ' || COALESCE(id::text, ''))
+) STORED;
+CREATE INDEX idx_adfs_strat_search_vector ON "adfs_strat" USING GIN (search_vector);
+
+ALTER TABLE "edge" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_edge_search_vector ON "edge" USING GIN (search_vector);
+
+ALTER TABLE "api_key" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_api_key_search_vector ON "api_key" USING GIN (search_vector);
+
+ALTER TABLE "oauth_applications" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(uid, '') || ' ' || COALESCE(redirect_uri, '') || ' ' ||
+    COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_oauth_applications_search_vector ON "oauth_applications" USING GIN (search_vector);
+
+ALTER TABLE "sets" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple', placeos_fts_join(keys) || ' ' || COALESCE(id::text, ''))
+) STORED;
+CREATE INDEX idx_sets_search_vector ON "sets" USING GIN (search_vector);
+
+ALTER TABLE "json_schema" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_json_schema_search_vector ON "json_schema" USING GIN (search_vector);
+
+ALTER TABLE "asset" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(identifier, '') || ' ' || COALESCE(serial_number, '') || ' ' ||
+    COALESCE(barcode, '') || ' ' || COALESCE(assigned_name, '') || ' ' || COALESCE(assigned_to, '') || ' ' ||
+    COALESCE(notes, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_asset_search_vector ON "asset" USING GIN (search_vector);
+
+ALTER TABLE "asset_type" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(brand, '') || ' ' || COALESCE(model_number, '') || ' ' ||
+    COALESCE(description, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_asset_type_search_vector ON "asset_type" USING GIN (search_vector);
+
+ALTER TABLE "asset_category" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_asset_category_search_vector ON "asset_category" USING GIN (search_vector);
+
+ALTER TABLE "asset_purchase_order" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(purchase_order_number, '') || ' ' || COALESCE(invoice_number, '') || ' ' ||
+    COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_asset_purchase_order_search_vector ON "asset_purchase_order" USING GIN (search_vector);
+
+ALTER TABLE "alert" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_alert_search_vector ON "alert" USING GIN (search_vector);
+
+ALTER TABLE "alert_dashboard" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_alert_dashboard_search_vector ON "alert_dashboard" USING GIN (search_vector);
+
+ALTER TABLE "shortener" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(uri, '') || ' ' || COALESCE(description, '') || ' ' ||
+    COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_shortener_search_vector ON "shortener" USING GIN (search_vector);
+
+ALTER TABLE "signage_plugin" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(uri, '') || ' ' ||
+    COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_signage_plugin_search_vector ON "signage_plugin" USING GIN (search_vector);
+
+ALTER TABLE "pending_mail" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS (
+  to_tsvector('simple',
+    placeos_fts_join(template) || ' ' || COALESCE(rejected_reason, '') || ' ' ||
+    placeos_fts_email(placeos_fts_join(send_to)) || ' ' || placeos_fts_email(send_from) || ' ' ||
+    COALESCE(id::text, '')
+  )
+) STORED;
+CREATE INDEX idx_pending_mail_search_vector ON "pending_mail" USING GIN (search_vector);
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+DROP INDEX IF EXISTS idx_sys_search_vector;
+DROP INDEX IF EXISTS idx_mod_search_vector;
+DROP INDEX IF EXISTS idx_driver_search_vector;
+DROP INDEX IF EXISTS idx_zone_search_vector;
+DROP INDEX IF EXISTS idx_user_search_vector;
+DROP INDEX IF EXISTS idx_repo_search_vector;
+DROP INDEX IF EXISTS idx_trigger_search_vector;
+DROP INDEX IF EXISTS idx_authority_search_vector;
+DROP INDEX IF EXISTS idx_ldap_strat_search_vector;
+DROP INDEX IF EXISTS idx_oauth_strat_search_vector;
+DROP INDEX IF EXISTS idx_adfs_strat_search_vector;
+DROP INDEX IF EXISTS idx_edge_search_vector;
+DROP INDEX IF EXISTS idx_api_key_search_vector;
+DROP INDEX IF EXISTS idx_oauth_applications_search_vector;
+DROP INDEX IF EXISTS idx_sets_search_vector;
+DROP INDEX IF EXISTS idx_json_schema_search_vector;
+DROP INDEX IF EXISTS idx_asset_search_vector;
+DROP INDEX IF EXISTS idx_asset_type_search_vector;
+DROP INDEX IF EXISTS idx_asset_category_search_vector;
+DROP INDEX IF EXISTS idx_asset_purchase_order_search_vector;
+DROP INDEX IF EXISTS idx_alert_search_vector;
+DROP INDEX IF EXISTS idx_alert_dashboard_search_vector;
+DROP INDEX IF EXISTS idx_shortener_search_vector;
+DROP INDEX IF EXISTS idx_signage_plugin_search_vector;
+DROP INDEX IF EXISTS idx_pending_mail_search_vector;
+
+ALTER TABLE "sys" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "mod" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "driver" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "zone" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "user" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "repo" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "trigger" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "authority" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "ldap_strat" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "oauth_strat" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "adfs_strat" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "edge" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "api_key" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "oauth_applications" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "sets" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "json_schema" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "asset" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "asset_type" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "asset_category" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "asset_purchase_order" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "alert" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "alert_dashboard" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "shortener" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "signage_plugin" DROP COLUMN IF EXISTS search_vector;
+ALTER TABLE "pending_mail" DROP COLUMN IF EXISTS search_vector;
+
+DROP FUNCTION IF EXISTS placeos_fts_email(text);
+DROP FUNCTION IF EXISTS placeos_fts_join(text[]);

spec/search_vector_spec.cr

@@ -0,0 +1,101 @@
+require "./helper"
+
+# PPT-2644: generated `search_vector` tsvector columns replacing Elasticsearch.
+# The query pattern pinned here (parameterized `to_tsquery('simple', ?)` with
+# `:*` prefix tokens joined by `&`) is what rest-api's index routes use.
+def vector_search(model, query : String)
+  model.where("search_vector @@ to_tsquery('simple', ?)", query).to_a
+end
+
+module PlaceOS::Model
+  describe "search_vector columns" do
+    Spec.after_each do
+      Module.clear
+      Driver.clear
+      ControlSystem.clear
+      Zone.clear
+      User.clear
+      Settings.clear
+    end
+
+    it "matches control systems on name, description and features with prefix tokens" do
+      cs = Generator.control_system
+      cs.name = "Video Conference Alpha"
+      cs.description = "boardroom telepresence"
+      cs.features = Set{"projector", "whiteboard"}
+      cs.save!
+
+      other = Generator.control_system
+      other.name = "Lobby Display"
+      other.save!
+
+      vector_search(ControlSystem, "video:* & conf:*").map(&.id).should eq [cs.id]
+      vector_search(ControlSystem, "telepres:*").map(&.id).should eq [cs.id]
+      # array columns (features) are searchable
+      vector_search(ControlSystem, "projector:*").map(&.id).should eq [cs.id]
+      # ids are searchable (Backoffice sends `id` in every fields list); the
+      # "sys" token comes only from the id prefix, so it matches both rows
+      vector_search(ControlSystem, "sys:*").size.should eq 2
+      vector_search(ControlSystem, "nonexistenttoken:*").should be_empty
+    end
+
+    it "matches users on tokenized email but never on secrets" do
+      user = Generator.user
+      user.email = Email.new("john.doe@example.com")
+      user.password_digest = "supersecrethash"
+      user.save!
+
+      # split tokens of the email address match…
+      vector_search(User, "john:* & doe:*").map(&.id).should eq [user.id]
+      vector_search(User, "example:*").map(&.id).should eq [user.id]
+      # …and the raw address is kept as a single lexeme too
+      vector_search(User, "john.doe@example.com").map(&.id).should eq [user.id]
+      # secrets are deliberately not indexed
+      vector_search(User, "supersecrethash:*").should be_empty
+    end
+
+    it "matches zones on tags" do
+      zone = Generator.zone
+      zone.name = "Level Three"
+      zone.tags = Set{"level", "building-a"}
+      zone.save!
+
+      vector_search(Zone, "level:* & building:*").map(&.id).should eq [zone.id]
+    end
+
+    it "matches modules on custom name, and drivers on name for join-based search" do
+      driver = Generator.driver
+      driver.name = "Cisco Video Switcher"
+      driver.save!
+
+      mod = Generator.module(driver: driver)
+      mod.custom_name = "Projector Left"
+      mod.save!
+
+      vector_search(Module, "projector:*").map(&.id).should eq [mod.id]
+      # module search by driver name is a query-time join (rest-api concern);
+      # the driver side of that join matches here
+      vector_search(Driver, "cisco:* & switch:*").map(&.id).should eq [driver.id]
+
+      # the EXISTS pattern rest-api uses for parent-child parity
+      Module
+        .where(
+          "(search_vector @@ to_tsquery('simple', ?) OR EXISTS (SELECT 1 FROM driver d WHERE d.id = driver_id AND d.search_vector @@ to_tsquery('simple', ?)))",
+          "cisco:*", "cisco:*"
+        ).to_a.map(&.id).should eq [mod.id]
+    end
+
+    it "matches settings on keys but not on the settings body" do
+      settings = Generator.settings(
+        settings_string: %({"api_secret_key": "hunter2"}),
+        encryption_level: Encryption::Level::None,
+      )
+      settings.save!
+
+      # NOTE: Settings versioning writes a history row on save, so match on
+      # inclusion rather than equality
+      vector_search(Settings, "api_secret_key:*").map(&.id).should contain(settings.id)
+      vector_search(Settings, "hunter2:*").should be_empty
+    end
+  end
+end