dunkirk.sh
pstream is dead; long live pstream
taciturnaxolotl.github.io/pstream-ng/
Add fuzzy search - Merge pull request #107 from Duplicake-fyi/production
+98
-3
src/backend/metadata/search.ts
+98
-3
src/backend/metadata/search.ts
···
1
+
import Fuse from "fuse.js";
2
+
1
3
import { SimpleCache } from "@/utils/cache";
2
4
import { MediaItem } from "@/utils/mediaTypes";
3
5
···
8
10
getMediaPoster,
9
11
multiSearch,
10
12
} from "./tmdb";
11
-
import { TMDBContentTypes } from "./types/tmdb";
13
+
import {
14
+
TMDBContentTypes,
15
+
TMDBMovieSearchResult,
16
+
TMDBShowSearchResult,
17
+
} from "./types/tmdb";
12
18
13
19
export interface MWQuery {
14
20
searchQuery: string;
···
22
28
23
29
// detect "tmdb:123456" or "tmdb:123456:movie" or "tmdb:123456:tv"
24
30
const tmdbIdPattern = /^tmdb:(\d+)(?::(movie|tv))?$/i;
31
+
const trailingYearPattern = /\s+\b(19|20)\d{2}\b$/;
32
+
33
+
function normalizeQuery(input: string): string {
34
+
return input
35
+
.toLowerCase()
36
+
.replace(/[^\p{L}\p{N}\s]/gu, " ")
37
+
.replace(/\s+/g, " ")
38
+
.trim();
39
+
}
40
+
41
+
function getLenientQueries(searchQuery: string): string[] {
42
+
const base = searchQuery.trim();
43
+
if (base.length < 3) return [base];
44
+
45
+
const normalized = normalizeQuery(base);
46
+
const withoutTrailingYear = base.replace(trailingYearPattern, "").trim();
47
+
const normalizedWithoutYear = normalizeQuery(withoutTrailingYear);
48
+
49
+
const variants = [
50
+
...new Set([base, normalized, withoutTrailingYear, normalizedWithoutYear]),
51
+
].filter((q) => q.length > 0);
52
+
53
+
// Keep fanout small to avoid TMDB rate-limit pressure.
54
+
return variants.slice(0, 2);
55
+
}
56
+
57
+
function dedupeTMDBResults(
58
+
items: (TMDBMovieSearchResult | TMDBShowSearchResult)[],
59
+
): (TMDBMovieSearchResult | TMDBShowSearchResult)[] {
60
+
const deduped = new Map<
61
+
string,
62
+
TMDBMovieSearchResult | TMDBShowSearchResult
63
+
>();
64
+
65
+
items.forEach((item) => {
66
+
deduped.set(`${item.media_type}:${item.id}`, item);
67
+
});
68
+
69
+
return Array.from(deduped.values());
70
+
}
71
+
72
+
function rankTMDBResultsFuzzy(
73
+
items: (TMDBMovieSearchResult | TMDBShowSearchResult)[],
74
+
query: string,
75
+
): (TMDBMovieSearchResult | TMDBShowSearchResult)[] {
76
+
if (items.length <= 1) return items;
77
+
78
+
const fuse = new Fuse(items, {
79
+
includeScore: true,
80
+
ignoreLocation: true,
81
+
threshold: 0.45,
82
+
minMatchCharLength: 2,
83
+
keys: [
84
+
{ name: "title", weight: 0.6 },
85
+
{ name: "name", weight: 0.6 },
86
+
{ name: "original_title", weight: 0.2 },
87
+
{ name: "original_name", weight: 0.2 },
88
+
],
89
+
});
90
+
91
+
const ranked = fuse.search(query).map((result) => result.item);
92
+
const rankedSet = new Set(
93
+
ranked.map((item) => `${item.media_type}:${item.id}`),
94
+
);
95
+
const remainder = items.filter(
96
+
(item) => !rankedSet.has(`${item.media_type}:${item.id}`),
97
+
);
98
+
99
+
return ranked.concat(remainder);
100
+
}
25
101
26
102
export async function searchForMedia(query: MWQuery): Promise<MediaItem[]> {
27
103
if (cache.has(query)) return cache.get(query) as MediaItem[];
···
69
145
}
70
146
}
71
147
72
-
const data = await multiSearch(searchQuery);
148
+
const queryVariants = getLenientQueries(searchQuery);
149
+
const settledResults = await Promise.allSettled(
150
+
queryVariants.map((q) => multiSearch(q)),
151
+
);
152
+
const fulfilledResults = settledResults
153
+
.filter(
154
+
(
155
+
result,
156
+
): result is PromiseFulfilledResult<
157
+
(TMDBMovieSearchResult | TMDBShowSearchResult)[]
158
+
> => result.status === "fulfilled",
159
+
)
160
+
.map((result) => result.value);
161
+
162
+
if (fulfilledResults.length === 0) {
163
+
return [];
164
+
}
165
+
166
+
const data = dedupeTMDBResults(fulfilledResults.flat());
167
+
const rankedData = rankTMDBResultsFuzzy(data, searchQuery);
73
168
74
-
const results = data.map((v) => {
169
+
const results = rankedData.map((v) => {
75
170
const formattedResult = formatTMDBSearchResult(v, v.media_type);
76
171
return formatTMDBMetaToMediaItem(formattedResult);
77
172
});
+1
-1
src/components/form/SearchBar.tsx
+1
-1
src/components/form/SearchBar.tsx
+2
src/hooks/useSearchQuery.ts
+2
src/hooks/useSearchQuery.ts
+38
-11
src/pages/parts/search/SearchListPart.tsx
+38
-11
src/pages/parts/search/SearchListPart.tsx
···
1
-
import { useEffect, useState } from "react";
1
+
import { useEffect, useRef, useState } from "react";
2
2
import { useTranslation } from "react-i18next";
3
3
import { useNavigate } from "react-router-dom";
4
-
import { useAsyncFn } from "react-use";
5
4
6
5
import { searchForMedia } from "@/backend/metadata/search";
7
6
import { MWQuery } from "@/backend/metadata/types/mw";
···
10
9
import { SectionHeading } from "@/components/layout/SectionHeading";
11
10
import { MediaGrid } from "@/components/media/MediaGrid";
12
11
import { WatchedMediaCard } from "@/components/media/WatchedMediaCard";
12
+
import { useDebounce } from "@/hooks/useDebounce";
13
13
import { Button } from "@/pages/About";
14
14
import { SearchLoadingPart } from "@/pages/parts/search/SearchLoadingPart";
15
15
import { MediaItem } from "@/utils/mediaTypes";
···
67
67
const { t } = useTranslation();
68
68
69
69
const [results, setResults] = useState<MediaItem[]>([]);
70
-
const [state, exec] = useAsyncFn((query: MWQuery) => searchForMedia(query));
70
+
const [loading, setLoading] = useState(false);
71
+
const [failed, setFailed] = useState(false);
72
+
const requestIdRef = useRef(0);
73
+
const debouncedSearchQuery = useDebounce(searchQuery, 300);
71
74
72
75
useEffect(() => {
73
-
async function runSearch(query: MWQuery) {
74
-
const searchResults = await exec(query);
75
-
if (!searchResults) return;
76
-
setResults(searchResults);
76
+
async function runSearch(query: MWQuery, requestId: number) {
77
+
setLoading(true);
78
+
setFailed(false);
79
+
80
+
let nextResults: MediaItem[] = [];
81
+
let didFail = false;
82
+
try {
83
+
nextResults = (await searchForMedia(query)) ?? [];
84
+
} catch {
85
+
didFail = true;
86
+
}
87
+
88
+
// Ignore stale responses from older requests.
89
+
if (requestIdRef.current !== requestId) {
90
+
return;
91
+
}
92
+
93
+
setFailed(didFail);
94
+
if (!didFail) setResults(nextResults);
95
+
setLoading(false);
77
96
}
78
97
79
-
if (searchQuery !== "") runSearch({ searchQuery });
80
-
}, [searchQuery, exec]);
98
+
if (debouncedSearchQuery === "") {
99
+
setResults([]);
100
+
setLoading(false);
101
+
setFailed(false);
102
+
return;
103
+
}
81
104
82
-
if (state.loading) return <SearchLoadingPart />;
83
-
if (state.error) return <SearchSuffix failed />;
105
+
requestIdRef.current += 1;
106
+
runSearch({ searchQuery: debouncedSearchQuery }, requestIdRef.current);
107
+
}, [debouncedSearchQuery]);
108
+
109
+
if (loading) return <SearchLoadingPart />;
110
+
if (failed) return <SearchSuffix failed />;
84
111
if (!results) return null;
85
112
86
113
return (
+21
-9
src/utils/cache.ts
+21
-9
src/utils/cache.ts
···
7
7
8
8
protected _storage: { key: Key; value: Value; expiry: Date }[] = [];
9
9
10
+
private static isExpired(entry: { expiry: Date }): boolean {
11
+
return entry.expiry.getTime() <= Date.now();
12
+
}
13
+
14
+
private pruneExpired(): void {
15
+
this._storage = this._storage.filter(
16
+
(entry) => !SimpleCache.isExpired(entry),
17
+
);
18
+
}
19
+
10
20
/*
11
21
** initialize store, will start the interval
12
22
*/
13
23
public initialize(): void {
14
24
if (this._interval) throw new Error("cache is already initialized");
15
25
this._interval = setInterval(() => {
16
-
const now = new Date();
17
-
this._storage.filter((val) => {
18
-
if (val.expiry < now) return false; // remove if expiry date is in the past
19
-
return true;
20
-
});
26
+
this.pruneExpired();
21
27
}, this.INTERVAL_MS);
22
28
}
23
29
···
26
32
*/
27
33
public destroy(): void {
28
34
if (this._interval) clearInterval(this._interval);
35
+
this._interval = null;
29
36
this.clear();
30
37
}
31
38
···
48
55
*/
49
56
public get(key: Key): Value | undefined {
50
57
if (!this._compare) throw new Error("Compare function not set");
58
+
this.pruneExpired();
51
59
const foundValue = this._storage.find(
52
60
(item) => this._compare && this._compare(item.key, key),
53
61
);
54
62
if (!foundValue) return undefined;
63
+
if (SimpleCache.isExpired(foundValue)) {
64
+
this.remove(key);
65
+
return undefined;
66
+
}
55
67
return foundValue.value;
56
68
}
57
69
···
60
72
*/
61
73
public set(key: Key, value: Value, expirySeconds: number): void {
62
74
if (!this._compare) throw new Error("Compare function not set");
75
+
this.pruneExpired();
63
76
const foundValue = this._storage.find(
64
77
(item) => this._compare && this._compare(item.key, key),
65
78
);
···
86
99
*/
87
100
public remove(key: Key): void {
88
101
if (!this._compare) throw new Error("Compare function not set");
89
-
this._storage.filter((val) => {
90
-
if (this._compare && this._compare(val.key, key)) return false; // remove if compare is success
91
-
return true;
92
-
});
102
+
this._storage = this._storage.filter(
103
+
(val) => !(this._compare && this._compare(val.key, key)),
104
+
);
93
105
}
94
106
95
107
/*