Merge branch 'master' into custom_domain

Author: Soxasora

README.md

@@ -131,6 +131,24 @@ services:
 
 You can read more about [docker compose override files](https://docs.docker.com/compose/multiple-compose-files/merge/).
 
+#### Enabling semantic search
+
+To enable semantic search that uses text embeddings, run `./scripts/nlp-setup`.
+
+Before running `./scripts/nlp-setup`, ensure the following are true:
+
+- search is enabled in `COMPOSE_PROFILES`:
+
+    ```.env
+    COMPOSE_PROFILES=...,search,...
+    ```
+- The default opensearch index (default name=`item`) is created and done indexing. This should happen the first time you run `./sndev start`, but it may take a few minutes for indexing to complete.
+
+After `nlp-setup` is done, restart your containers to enable semantic search:
+
+```
+> ./sndev restart
+```
 
 
 <br>

api/paidAction/downZap.js

@@ -62,7 +62,7 @@ export async function onPaid ({ invoice, actId }, { tx }) {
   // denormalize downzaps
   await tx.$executeRaw`
     WITH territory AS (
-      SELECT COALESCE(r."subName", i."subName", 'meta')::TEXT as "subName"
+      SELECT COALESCE(r."subName", i."subName", 'meta')::CITEXT as "subName"
       FROM "Item" i
       LEFT JOIN "Item" r ON r.id = i."rootId"
       WHERE i.id = ${itemAct.itemId}::INTEGER

api/paidAction/zap.js

@@ -151,7 +151,7 @@ export async function onPaid ({ invoice, actIds }, { tx }) {
   // NOTE: for the rows that might be updated by a concurrent zap, we use UPDATE for implicit locking
   await tx.$queryRaw`
     WITH territory AS (
-      SELECT COALESCE(r."subName", i."subName", 'meta')::TEXT as "subName"
+      SELECT COALESCE(r."subName", i."subName", 'meta')::CITEXT as "subName"
       FROM "Item" i
       LEFT JOIN "Item" r ON r.id = i."rootId"
       WHERE i.id = ${itemAct.itemId}::INTEGER

api/resolvers/item.js

@@ -15,7 +15,6 @@ import {
   FULL_COMMENTS_THRESHOLD
 } from '@/lib/constants'
 import { msatsToSats } from '@/lib/format'
-import { parse } from 'tldts'
 import uu from 'url-unshort'
 import { actSchema, advSchema, bountySchema, commentSchema, discussionSchema, jobSchema, linkSchema, pollSchema, validateSchema } from '@/lib/validate'
 import { defaultCommentSort, isJob, deleteItemByAuthor } from '@/lib/item'
@@ -26,6 +25,7 @@ import assertApiKeyNotPermitted from './apiKey'
 import performPaidAction from '../paidAction'
 import { GqlAuthenticationError, GqlInputError } from '@/lib/error'
 import { verifyHmac } from './wallet'
+import { parse } from 'tldts'
 
 function commentsOrderByClause (me, models, sort) {
   const sharedSortsArray = []
@@ -613,7 +613,6 @@ export default {
       const urlObj = new URL(ensureProtocol(url))
       let { hostname, pathname } = urlObj
 
-      // remove subdomain from hostname
       const parseResult = parse(urlObj.hostname)
       if (parseResult?.subdomain?.length > 0) {
         hostname = hostname.replace(`${parseResult.subdomain}.`, '')
@@ -639,6 +638,9 @@ export default {
       } else if (urlObj.hostname === 'yewtu.be') {
         const matches = url.match(/(https?:\/\/)?yewtu\.be.*(v=|embed\/)(?<id>[_0-9a-z-]+)/i)
         similar = `^(http(s)?:\\/\\/)?yewtu\\.be\\/(watch\\?v\\=|embed\\/)${matches?.groups?.id}&?`
+      } else {
+        // only allow ending of mismatching search params
+        similar += '(?:\\?.*)?$'
       }
 
       return await itemQueryWithMeta({

awards.csv

@@ -205,3 +205,10 @@ cointastical,issue,#1962,#1217,good-first-issue,,,,2k,???,???
 Scroogey-SN,pr,#1975,#1964,good-first-issue,,,,20k,Scroogey@coinos.io,2025-04-02
 rideandslide,issue,#1986,#1985,good-first-issue,,,,2k,koiora@getalby.com,2025-04-02
 kristapsk,issue,#1976,#841,good-first-issue,,,,2k,???,???
+ed-kung,pr,#2070,#2061,good-first-issue,,,,20k,simplestacker@getalby.com,???
+ed-kung,issue,#2070,#2061,good-first-issue,,,,2k,simplestacker@getalby.com,???
+ed-kung,pr,#2070,#2058,easy,,,,100k,simplestacker@getalby.com,???
+ed-kung,pr,#2070,#2047,medium-hard,,,,500k,simplestacker@getalby.com,???
+SouthKoreaLN,pr,#2068,#2064,good-first-issue,,,,20k,south_korea_ln@stacker.news,???
+kepford,issue,#2068,#2064,good-first-issue,,,,2k,???,???
+SouthKoreaLN,pr,#2069,#1990,good-first-issue,,,,20k,south_korea_ln@stacker.news,???

components/item-act.js

@@ -18,7 +18,10 @@ import { useSendWallets } from '@/wallets/index'
 const defaultTips = [100, 1000, 10_000, 100_000]
 
 const Tips = ({ setOValue }) => {
-  const tips = [...getCustomTips(), ...defaultTips].sort((a, b) => a - b)
+  const customTips = getCustomTips()
+  const defaultNoCustom = defaultTips.filter(d => !customTips.includes(d))
+  const tips = [...customTips, ...defaultNoCustom].slice(0, 7).sort((a, b) => a - b)
+
   return tips.map((num, i) =>
     <Button
       size='sm'
@@ -37,11 +40,7 @@ const Tips = ({ setOValue }) => {
 const getCustomTips = () => JSON.parse(window.localStorage.getItem('custom-tips')) || []
 
 const addCustomTip = (amount) => {
-  if (defaultTips.includes(amount)) return
-  let customTips = Array.from(new Set([amount, ...getCustomTips()]))
-  if (customTips.length > 3) {
-    customTips = customTips.slice(0, 3)
-  }
+  const customTips = Array.from(new Set([amount, ...getCustomTips()])).slice(0, 7)
   window.localStorage.setItem('custom-tips', JSON.stringify(customTips))
 }
 

docker-compose.yml

@@ -176,31 +176,16 @@ services:
       - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}
       - plugins.security.disabled=true
       - discovery.type=single-node
+      - "_JAVA_OPTIONS=-Xms2g -Xmx2g -XX:UseSVE=0"
     ports:
       - 9200:9200 # REST API
       - 9600:9600 # Performance Analyzer
     volumes:
       - os:/usr/share/opensearch/data
+      - ./docker/opensearch/init-opensearch.sh:/usr/share/opensearch/init-opensearch.sh
     labels:
       CONNECT: "localhost:9200"
-    command: >
-      bash -c '
-        set -m
-        /usr/share/opensearch/opensearch-docker-entrypoint.sh &
-        until curl -sS "http://localhost:9200/_cat/health?h=status" -ku admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} | grep -q "green\|yellow"; do
-          echo "Waiting for OpenSearch to start..."
-          sleep 1
-        done
-        echo "OpenSearch started."
-        curl \
-          -H "Content-Type: application/json" \
-          -X PUT \
-          -d '{"mappings":{"properties":{"text":{"type":"text","analyzer":"english","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"title":{"type":"text","analyzer":"english","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}}' \
-          "http://localhost:9200/item" \
-          -ku admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD}
-        echo "OpenSearch index created."
-        fg
-      '
+    command: ["bash", "/usr/share/opensearch/init-opensearch.sh"]
     cpu_shares: "${CPU_SHARES_LOW}"
   os-dashboard:
     image: opensearchproject/opensearch-dashboards:2.17.0

docker/opensearch/init-opensearch.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -m
+
+/usr/share/opensearch/opensearch-docker-entrypoint.sh &
+
+# ---- Wait for OpenSearch to start
+
+until curl -sS "http://localhost:9200/_cat/health?h=status" -ku admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} | grep -q "green\|yellow"; do
+  echo "Waiting for OpenSearch to start..."
+  sleep 1
+done
+
+# ---- If index doesn't exist, create it with default settings
+
+index_exists=$(curl -s -o /dev/null -w "%{http_code}" -I "http://localhost:9200/$OPENSEARCH_INDEX")
+
+if [ "$index_exists" -eq 200 ]; then
+  echo "OpenSearch index $OPENSEARCH_INDEX already exists."
+else
+  curl \
+    -H "Content-Type: application/json" \
+    -X PUT \
+    -d '{
+      "mappings": {
+        "properties": {
+          "text": {
+            "type": "text",
+            "analyzer": "english",
+            "fields": {"keyword":{"type":"keyword","ignore_above":256}}
+          },
+          "title": {
+            "type": "text",
+            "analyzer": "english",
+            "fields": {"keyword":{"type":"keyword","ignore_above":256}}
+          }}}}' \
+    "http://localhost:9200/$OPENSEARCH_INDEX" \
+    -ku admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD}
+  echo ""
+  echo "OpenSearch index $OPENSEARCH_INDEX created."
+fi
+
+fg

docs/dev/semantic-search.md

@@ -1,4 +1,26 @@
-Getting semantic search setup in OpenSearch is currently a multistep, manual process. To configure semantic search, enter the following commands into OpenSearch's REST API. You can do this in Dev Tools in the OpenSearch Dashboard (after starting your SN dev environment, point your browser to localhost:5601). You can also use CURL to send these commands to localhost:9200.
+## Automated setup
+
+To enable semantic search that uses text embeddings, run `./scripts/nlp-setup`.
+
+Before running `./scripts/nlp-setup`, ensure the following are true:
+
+- search is enabled in `COMPOSE_PROFILES`:
+
+    ```.env
+    COMPOSE_PROFILES=...,search,...
+    ```
+- The default opensearch index (default name=`item`) is created and done indexing. This should happen the first time you run `./sndev start`, but it may take a few minutes for indexing to complete.
+
+After `nlp-setup` is done, restart your containers to enable semantic search:
+
+```
+> ./sndev restart
+```
+
+
+## Manual setup
+
+You can also set up and configure semantic search manually. To do so, enter the following commands into OpenSearch's REST API. You can do this in Dev Tools in the OpenSearch Dashboard (after starting your SN dev environment, point your browser to localhost:5601). You can also use CURL to send these commands to localhost:9200.
 
 ### step 1: configure the ml plugin
 ```json