Merge remote-tracking branch 'refs/remotes/origin/master'

Author: lu391see

client/README.md

@@ -1,39 +1,33 @@
-# client-vue
+# TueR – Tübingen Retrieval (Frontend)
 
-This template should help get you started developing with Vue 3 in Vite.
+This is the frontend for the **TüR** – **Tü**bingen **R**etrieval-project, built with Vue 3 and Vite.
 
-## Recommended IDE Setup
+## Requirements
 
-[VSCode](https://code.visualstudio.com/) + [Volar](https://marketplace.visualstudio.com/items?itemName=Vue.volar) (and disable Vetur).
+- [bun.sh](https://bun.sh) (the runtime environment for this project)
 
-## Type Support for `.vue` Imports in TS
+## Getting Started
 
-TypeScript cannot handle type information for `.vue` imports by default, so we replace the `tsc` CLI with `vue-tsc` for type checking. In editors, we need [Volar](https://marketplace.visualstudio.com/items?itemName=Vue.volar) to make the TypeScript language service aware of `.vue` types.
-
-## Customize configuration
-
-See [Vite Configuration Reference](https://vitejs.dev/config/).
-
-## Project Setup
+To install the dependencies, run:
 
 ```sh
 bun install
 ```
 
-### Compile and Hot-Reload for Development
+## Start the server and start opening doors to Tübingen
+
+To start the development server, run:
 
 ```sh
 bun dev
 ```
 
-### Type-Check, Compile and Minify for Production
+This starts the development server at <http://localhost:5173>.
 
-```sh
-bun build
-```
+---
 
-### Lint with [ESLint](https://eslint.org/)
+**Note**:
 
-```sh
-bun lint
-```
+Make sure to have the backend running as well. You can find the backend at [engine](../engine).
+
+---

client/src/components/SearchResult.vue

@@ -24,15 +24,18 @@
               class="px-3 py-1 bg-primary-100 text-primary-600 rounded-full text-xs font-medium">
           {{ tag }}
         </span>
-        <span class="px-3 py-1 bg-primary-100 text-primary-600 rounded-full text-xs font-medium" v-if="result.score">
+        <!-- <span class="px-3 py-1 bg-primary-100 text-primary-600 rounded-full text-xs font-medium" v-if="result.score">
           {{ Math.round((result.score ?? 0) * 100) / 100 }}%
-        </span>
+        </span> -->
       </div>
       <div class="mt-4">
-        <button @click="toggleSummary"
+        <button @click="toggleSummary" :disabled="loadingSummary"
                 class="text-sm text-primary-600 hover:text-primary-700 font-medium focus:outline-none">
           {{ showSummary ? 'Hide' : 'Show' }} Summary
         </button>
+        <div v-if="loadingSummary"
+             class="animate-spin rounded-full h-4 w-4 border-t-2 border-b-2 border-primary-500 mx-2">
+        </div>
       </div>
       <transition
         enter-active-class="transition ease-out duration-200"
@@ -74,12 +77,9 @@ interface SearchResultProps {
 
 const { result } = defineProps<SearchResultProps>()
 
-const showSummary = ref(false)
-const summary = ref('')
 
 const showPreview = ref(false)
 const preview = ref('')
-
 const togglePreview = async () => {
   showPreview.value = !showPreview.value
   if (preview.value) {
@@ -90,14 +90,23 @@ const togglePreview = async () => {
   preview.value = URL.createObjectURL(await response.blob())
 }
 
+const loadingSummary = ref(false)
+const showSummary = ref(false)
+const summary = ref('')
 const toggleSummary = async () => {
+  loadingSummary.value = true
   showSummary.value = !showSummary.value
-  if (result.summary) {
+  if (summary.value) {
+    loadingSummary.value = false
     return
   }
 
-  const response = await fetch(`${ENGINE_ENDPOINT}/summary/${result.id}`)
-  summary.value = await response.text()
+  summary.value = await fetch(`${ENGINE_ENDPOINT}/summary/${result.id}`)
+    .then((res) => res.json())
+    .then((data) => {
+      loadingSummary.value = false
+      return data.summary
+    })
 }
 
 //watch(() => result.url, downloadPreview, { immediate: true })

client/src/stores/search.ts

@@ -24,6 +24,7 @@ const dummyResults: SearchResult[] = [
 export const useSearchStore = defineStore('search', () => {
   const localStore = useLocal()
 
+  const lastQuery = ref('')
   const internalResults = ref<SearchResult[]>([])
   const results = computed(() => {
     const results = internalResults.value
@@ -35,11 +36,13 @@ export const useSearchStore = defineStore('search', () => {
 
   function search(query: string) {
     if (query === '') {
-      internalResults.value = dummyResults
+      internalResults.value = []
       return
     }
 
-    fetch(`${ENGINE_ENDPOINT}/search?query=${query}`)
+    lastQuery.value = query
+
+    return fetch(`${ENGINE_ENDPOINT}/search?query=${query}`)
       .then((response) => response.json())
       .then((data: ApiSearchResultsResponse) => {
         internalResults.value = data.results
@@ -51,6 +54,7 @@ export const useSearchStore = defineStore('search', () => {
   return {
     results,
     lastSearches,
+    lastQuery,
     search
   }
 })

client/src/views/SearchView.vue

@@ -38,6 +38,9 @@
           {{ lastSearch }}
         </span>
       </div>
+      <div v-if="searched && results.length > 0" class="text-gray-600 mt-8">
+        Found {{ results.length }} {{ results.length === 0 ? 'door' : 'doors' }} for "{{ lastQuery }}"
+      </div>
       <div v-if="loading" class="text-center text-gray-600 mt-8">
         <div class="animate-spin rounded-full h-12 w-12 border-t-2 border-b-2 border-tertiary-500 mx-auto"></div>
         <p class="mt-4">{{ TITLE }}</p>
@@ -56,7 +59,7 @@
           </li>
         </transition-group>
         <p v-else-if="searched && results.length === 0" class="text-center text-gray-600 mt-8">
-          No results found. Try another search term.
+          No doors found for "{{ lastQuery }}". Try another search term.
         </p>
       </div>
     </div>
@@ -73,15 +76,15 @@ import { gsap } from 'gsap'
 import SearchResult from '@/components/SearchResult.vue'
 
 const searchStore = useSearchStore()
-const { results, lastSearches } = storeToRefs(searchStore)
+const { results, lastSearches, lastQuery } = storeToRefs(searchStore)
 
 const loading = false
 
 const query = ref('')
 const searched = ref(false)
 const search = () => {
-  searched.value = true
-  searchStore.search(query.value)
+  searched.value = false
+  searchStore.search(query.value)?.then(() => searched.value = true)
 }
 
 

engine/README.md

@@ -1,25 +1,21 @@
-# Search Engine
+# TueR – Tübingen Retrieval (Engine)
 
-## Usage
-
-- To crawl pages, you need to run the following command:
+This is the engine for the **TüR** (**Tü**bingen **R**etrieval) project, built with Python, Flask, DuckDB, and lots of
+motivation.
 
-   ```
-   python main.py --online
-   ```
-- Run the following command to start the server:
-
-   ```
-   python server.py
-   ```
+## Table of Contents
 
-- Open your browser and navigate to `http://localhost:8000/` to view the application.
+1. [Requirements](#requirements)
+2. [Installation](#installation)
+3. [Usage](#usage)
+4. [Server](#server)
+5. [Known Issues](#known-issues)
 
 ## Requirements
 
-- `Python 3`
-- `pip`
-- `virtualenv`
+- Python 3
+- pip
+- virtualenv
 
 ## Installation
 
@@ -28,40 +24,64 @@
 - Download and install the latest version of Python 3 from the official website.
 
 2. **Install virtualenv:**
+   ```shell
+   pip install virtualenv
+   ```
 
-- Open a terminal and run the following command:
-
+3. **Create and activate a virtual environment:**
+   ```shell
+   virtualenv --python=3.11 .venv
+   source .venv/bin/activate
    ```
-   pip install virtualenv
+
+4. **Install requirements:**
+   ```shell
+   pip install -r requirements.txt
+   python -m spacy download en_core_web_sm
    ```
 
-3. **Create a virtual environment:**
+## Usage
 
-- Create the virtual environment:
+### Crawl pages:
 
-  ```
-  virtualenv --python=3.11 .venv
-  ```
+```shell
+python main.py --online
+```
 
-- Activate the virtual environment:
+### Start the server:
 
-  ```
-  source .venv/bin/activate
-  ```
+```shell
+python server.py
+```
 
-4. **Install requirements:**
+### Access the application:
 
-- Ensure you have a requirements.txt file in your project directory.
-- Run the following command to install the dependencies:
+Open your browser and navigate to [http://localhost:8000/](http://localhost:8000/)
 
-   ```
-   pip install -r requirements.txt
-   ```
+## Server
 
-- For the text-processing part, we use `spaCy`. You need to download the English model by running the following command:
+The server is built with Flask and runs on port 8000 by default. To start the server, use the following command:
 
-   ```
-    python -m spacy download en_core_web_sm
-   ```
+```shell
+python server.py
+```
+
+You can see a list of all available routes by navigating to <http://localhost:8000/site-map>.
+
+---
+
+**Important:**
+
+- The server will only work if you have crawled some pages before.
+- For the summarization you will need a strong CPU and a lot of RAM, as the summarization is done on the fly and can be
+  quite resource-intensive.
+---
+
+## Known Issues
+
+The pipeline will not stop by itself, even if reached the maximum sites.
+You will have to stop it manually by pressing `Ctrl + C` in the terminal.
+But it will be able to resume from where it left off when you restart it.
 
-5. **Start developing the project**
+When the offline pipeline runs, it will try to finish completely before stopping.
+If you force stop it, the pipeline will not save the state because it's saved in `crawlies.db.wal`.

engine/server.py

@@ -112,14 +112,14 @@ def summarize(doc_id):
 def doc_count():
     con = dbcon.cursor()
 
+    # Get the number of documents
     doc_count = con.execute(
         """
         SELECT COUNT(*) FROM documents
     """
     ).fetchall()[0][0]
 
     con.cursor()
-
     return jsonify({"doc_count": doc_count})
 
 

engine/similarity.py

@@ -17,7 +17,7 @@ def most_similar(word: str, topn=7) -> list:
 
     Args:
         word (str): The word to find similar words to.
-        topn (int, optional): Amount of similiar words we want to have. Defaults to 7.
+        topn (int, optional): Number of similar words we want to have. Default to 7.
 
     Returns:
         list: A list of tuples containing the most similar words and their similarity score.