tsiry-sandratraina.com
A modern Music Player Daemon based on Rockbox open source high quality audio player
libadwaita
audio
rust
zig
deno
mpris
rockbox
mpd
1/***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 *
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
19 *
20 ****************************************************************************/
21
22/*
23 * TagCache API
24 *
25 * ----------x---------x------------------x-----
26 * | | | External
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
30 * | (R/W) | | | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
36 * | | (R) |
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
55 *
56 */
57
58#if !defined(PLUGIN)
59
60/*#define LOGF_ENABLE*/
61/*#define LOGF_CLAUSES define to enable logf clause matching (LOGF_ENABLE req'd) */
62
63#include <stdio.h>
64#include <stdlib.h>
65#include <ctype.h>
66#ifdef APPLICATION
67#include <unistd.h> /* readlink() */
68#include <limits.h> /* PATH_MAX */
69#endif
70#include "config.h"
71#include "ata_idle_notify.h"
72#include "thread.h"
73#include "kernel.h"
74#include "system.h"
75#include "logf.h"
76#include "string-extra.h"
77#include "usb.h"
78#include "metadata.h"
79#include "tagcache.h"
80#include "yesno.h"
81#include "core_alloc.h"
82#include "crc32.h"
83#include "misc.h"
84#include "settings.h"
85#include "dir.h"
86#include "pathfuncs.h"
87#include "debug.h"
88#include "dircache.h"
89#include "errno.h"
90
91#ifndef __PCTOOL__
92#include "lang.h"
93#include "eeprom_settings.h"
94#endif
95#define USR_CANCEL false
96#else/*!defined(PLUGIN)*/
97#define USR_CANCEL (tc_stat.commit_delayed == true)
98#endif /*!defined(PLUGIN)*/
99/*
100 * Define this to support non-native endian tagcache files.
101 * Databases are always written in native endian so this is
102 * basically only necessary to support databases generated
103 * by the PC database tool.
104 *
105 * Adds around 0.5-1.0k of code.
106 */
107#define TAGCACHE_SUPPORT_FOREIGN_ENDIAN
108
109/* Allow a little drift to the filename ordering (should not be too high/low). */
110#define POS_HISTORY_COUNT 4
111
112/* How much to pre-load entries while committing to prevent seeking. */
113#define IDX_BUF_DEPTH 64
114
115/* Tag Cache Header version 'TCHxx'. Increment when changing internal structures. */
116#define TAGCACHE_MAGIC 0x54434810
117
118/* Dump store/restore header version 'TCSxx'. */
119#define TAGCACHE_STATEFILE_MAGIC 0x54435301
120
121/* How much to allocate extra space for ramcache. */
122#define TAGCACHE_RESERVE 32768
123
124/*
125 * Define how long one entry must be at least (longer -> less memory at commit).
126 * Must be at least 4 bytes in length for correct alignment.
127 */
128#define TAGFILE_ENTRY_CHUNK_LENGTH 8
129
130/* Used to guess the necessary buffer size at commit. */
131#define TAGFILE_ENTRY_AVG_LENGTH 16
132
133/* Max events in the internal tagcache command queue. */
134#define TAGCACHE_COMMAND_QUEUE_LENGTH 32
135
136/* Idle time before committing events in the command queue. */
137#define TAGCACHE_COMMAND_QUEUE_COMMIT_DELAY HZ*2
138
139/* Dont commit database_tmp data. */
140#define TAGCACHE_FILE_NOCOMMIT "database_commit.ignore"
141
142/* Temporary database containing new tags to be committed to the main db. */
143#define TAGCACHE_FILE_TEMP "database_tmp.tcd"
144
145/* The main database master index and numeric data. */
146#define TAGCACHE_FILE_MASTER "database_idx.tcd"
147
148/* The main database string data. */
149#define TAGCACHE_FILE_INDEX "database_%d.tcd"
150
151/* ASCII dumpfile of the DB contents. */
152#define TAGCACHE_FILE_CHANGELOG "database_changelog.txt"
153
154/* Serialized DB. */
155#define TAGCACHE_STATEFILE "database_state.tcd"
156
157/* Flags */
158#define FLAG_DELETED 0x0001 /* Entry has been removed from db */
159#define FLAG_DIRCACHE 0x0002 /* Filename is a dircache pointer */
160#define FLAG_DIRTYNUM 0x0004 /* Numeric data has been modified */
161#define FLAG_TRKNUMGEN 0x0008 /* Track number has been generated */
162#define FLAG_RESURRECTED 0x0010 /* Statistics data has been resurrected */
163
164#ifdef __PCTOOL__
165#define yield() do { } while(0)
166#define sim_sleep(timeout) do { } while(0)
167#define do_timed_yield() do { } while(0)
168static void db_log(const char *prefix, const char *msg);
169#endif
170
171#ifdef __PCTOOL__
172#define DB_LOG(__prefix, __file) db_log(__prefix, __file)
173#else
174#define DB_LOG(...)
175#endif
176
177#ifndef __PCTOOL__
178/* Tag Cache thread. */
179static struct event_queue tagcache_queue SHAREDBSS_ATTR;
180static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
181static const char tagcache_thread_name[] = "tagcache";
182#endif
183
184/* Previous path when scanning directory tree recursively. */
185static char curpath[TAGCACHE_BUFSZ];
186/* Shared buffer for several build_index fns to reduce stack usage */
187static char build_idx_buf[TAGCACHE_BUFSZ];
188static const long build_idx_bufsz = sizeof(build_idx_buf);
189/* Used when removing duplicates. */
190static char *tempbuf; /* Allocated when needed. */
191static long tempbufidx; /* Current location in buffer. */
192static size_t tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
193static long tempbuf_left; /* Buffer space left. */
194static long tempbuf_pos;
195#ifndef __PCTOOL__
196static int tempbuf_handle;
197#endif
198
199#define SORTED_TAGS_COUNT 9
200#define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
201#define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
202#define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
203 (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
204/* Tags we want to get sorted (loaded to the tempbuf). */
205#define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
206 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
207 (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title) | \
208 (1LU << tag_virt_canonicalartist))
209
210/* Uniqued tags (we can use these tags with filters and conditional clauses). */
211#define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
212 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
213 (1LU << tag_albumartist) | (1LU << tag_grouping) | \
214 (1LU << tag_virt_canonicalartist))
215
216/* String presentation of the tags defined in tagcache.h. Must be in correct order! */
217static const char * const tags_str[] = { "artist", "album", "genre", "title",
218 "filename", "composer", "comment", "albumartist", "grouping", "year",
219 "discnumber", "tracknumber", "canonicalartist", "bitrate", "length",
220 "playcount", "rating", "playtime", "lastplayed", "commitid", "mtime",
221 "lastelapsed", "lastoffset"
222#if !defined(LOGF_ENABLE) || !defined(LOGF_CLAUSES)
223};
224#define logf_clauses(...) do { } while(0)
225#else /* strings for logf debugging */
226 "tag_virt_basename", "tag_virt_length_min", "tag_virt_length_sec",
227 "tag_virt_playtime_min", "tag_virt_playtime_sec",
228 "tag_virt_entryage", "tag_virt_autoscore"
229};
230/* more debug strings */
231static const char * const tag_type_str[] = {
232 [clause_none] = "clause_none", [clause_is] = "clause_is",
233 [clause_is_not] = "clause_is_not", [clause_gt] = "clause_gt",
234 [clause_gteq] = "clause_gteq", [clause_lt] = "clause_lt",
235 [clause_lteq] = "clause_lteq", [clause_contains] = "clause_contains",
236 [clause_not_contains] = "clause_not_contains",
237 [clause_begins_with] = "clause_begins_with",
238 [clause_not_begins_with] = "clause_not_begins_with",
239 [clause_ends_with] = "clause_ends_with",
240 [clause_not_ends_with] = "clause_not_ends_with",
241 [clause_oneof] = "clause_oneof",
242 [clause_begins_oneof] = "clause_begins_oneof",
243 [clause_ends_oneof] = "clause_ends_oneof",
244 [clause_not_oneof] = "clause_not_oneof",
245 [clause_not_begins_oneof] = "clause_not_begins_oneof",
246 [clause_not_ends_oneof] = "clause_not_ends_oneof",
247 [clause_logical_or] = "clause_logical_or"
248 };
249#define logf_clauses logf
250#endif /* !defined(LOGF_ENABLE) || !defined(LOGF_CLAUSES) */
251
252#if defined(PLUGIN)
253char *itoa_buf(char *buf, size_t bufsz, long int i)
254{
255 snprintf(buf, bufsz, "%ld", i);
256 return buf;
257}
258#endif
259
260/* Status information of the tagcache. */
261static struct tagcache_stat tc_stat;
262
263/* Queue commands. */
264enum tagcache_queue {
265 Q_STOP_SCAN = 0,
266 Q_START_SCAN,
267 Q_IMPORT_CHANGELOG,
268 Q_UPDATE,
269 Q_REBUILD,
270
271 /* Internal tagcache command queue. */
272 CMD_UPDATE_MASTER_HEADER,
273 CMD_UPDATE_NUMERIC,
274};
275
276struct tagcache_command_entry {
277 int32_t command;
278 int32_t idx_id;
279 int32_t tag;
280 int32_t data;
281};
282
283#ifndef __PCTOOL__
284static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
285static volatile int command_queue_widx = 0;
286static volatile int command_queue_ridx = 0;
287static struct mutex command_queue_mutex SHAREDBSS_ATTR;
288#endif
289
290/* Tag database structures. */
291
292/* Variable-length tag entry in tag files. */
293struct tagfile_entry {
294 int32_t tag_length; /* Length of the data in bytes including '\0' */
295 int32_t idx_id; /* Corresponding entry location in index file of not unique tags */
296 char tag_data[0]; /* Begin of the tag data */
297};
298
299/* Fixed-size tag entry in master db index. */
300struct index_entry {
301 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
302 int32_t flag; /* Status flags */
303};
304
305/* Header is the same in every file. */
306struct tagcache_header {
307 int32_t magic; /* Header version number */
308 int32_t datasize; /* Data size in bytes */
309 int32_t entry_count; /* Number of entries in this file */
310};
311
312struct master_header {
313 struct tagcache_header tch;
314 int32_t serial; /* Increasing counting number */
315 int32_t commitid; /* Number of commits so far */
316 int32_t dirty;
317};
318
319static struct master_header current_tcmh;
320
321#ifdef HAVE_TC_RAMCACHE
322
323#define TC_ALIGN_PTR(p, type, gap_out_p) \
324 ({ typeof (p) __p = (p); \
325 typeof (p) __palgn = ALIGN_UP(__p, __alignof__(type)); \
326 *(gap_out_p) = (char *)__palgn - (char *)__p; \
327 __palgn; })
328
329#define IF_TCRCDC(...) IF_DIRCACHE(__VA_ARGS__)
330
331#ifdef HAVE_DIRCACHE
332#define tcrc_dcfrefs \
333 ((struct dircache_fileref *)(tcramcache.hdr->tags[tag_filename] + \
334 sizeof (struct tagcache_header)))
335#endif /* HAVE_DIRCACHE */
336
337/* Header is created when loading database to ram. */
338struct ramcache_header {
339 char *tags[TAG_COUNT]; /* Tag file content (dcfrefs if tag_filename) */
340 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
341 struct index_entry indices[0]; /* Master index file content */
342};
343
344#ifdef HAVE_EEPROM_SETTINGS
345struct statefile_header {
346 int32_t magic; /* Statefile version number */
347 struct master_header mh; /* Header from the master index */
348 struct ramcache_header *hdr; /* Old load address of hdr for relocation */
349 struct tagcache_stat tc_stat;
350};
351#endif /* HAVE_EEPROM_SETTINGS */
352
353/* In-RAM ramcache structure (not persisted) */
354static struct tcramcache
355{
356 struct ramcache_header *hdr; /* allocated ramcache_header */
357 int handle; /* buffer handle */
358} tcramcache;
359
360static inline void tcrc_buffer_lock(void)
361{
362 core_pin(tcramcache.handle);
363}
364
365static inline void tcrc_buffer_unlock(void)
366{
367 core_unpin(tcramcache.handle);
368}
369
370#else /* ndef HAVE_TC_RAMCACHE */
371
372#define IF_TCRCDC(...)
373
374#endif /* HAVE_TC_RAMCACHE */
375
376/**
377 * Full tag entries stored in a temporary file waiting
378 * for commit to the cache. */
379struct temp_file_entry {
380 int32_t tag_offset[TAG_COUNT];
381 int16_t tag_length[TAG_COUNT];
382 int32_t flag;
383 int32_t data_length;
384};
385
386struct tempbuf_id_list {
387 long id;
388 struct tempbuf_id_list *next;
389};
390
391struct tempbuf_searchidx {
392 long idx_id;
393 char *str;
394 int seek;
395 struct tempbuf_id_list idlist;
396};
397
398/* Lookup buffer for fixing messed up index while after sorting. */
399static long commit_entry_count;
400static long lookup_buffer_depth;
401static struct tempbuf_searchidx **lookup;
402
403/* Used when building the temporary file. */
404static int cachefd = -1, filenametag_fd;
405static int total_entry_count = 0;
406static int data_size = 0;
407static int processed_dir_count;
408
409/* Thread safe locking */
410static volatile int write_lock;
411static volatile int read_lock;
412
413static bool delete_entry(long idx_id);
414
415static inline void str_setlen(char *buf, size_t len)
416{
417 buf[len] = '\0';
418}
419
420const char* tagcache_tag_to_str(int tag)
421{
422 return tags_str[tag];
423}
424
425#ifdef TAGCACHE_SUPPORT_FOREIGN_ENDIAN
426static void swap_tagfile_entry(struct tagfile_entry *buf)
427{
428 if (tc_stat.econ)
429 {
430 buf->tag_length = swap32(buf->tag_length);
431 buf->idx_id = swap32(buf->idx_id);
432 }
433}
434
435static void swap_index_entry(struct index_entry *buf)
436{
437 if (tc_stat.econ)
438 {
439 for (int i = 0; i < TAG_COUNT; ++i)
440 buf->tag_seek[i] = swap32(buf->tag_seek[i]);
441 buf->flag = swap32(buf->flag);
442 }
443}
444
445static void swap_tagcache_header(struct tagcache_header *buf)
446{
447 if (tc_stat.econ)
448 {
449 buf->magic = swap32(buf->magic);
450 buf->datasize = swap32(buf->datasize);
451 buf->entry_count = swap32(buf->entry_count);
452 }
453}
454
455static void swap_master_header(struct master_header *buf)
456{
457 if (tc_stat.econ)
458 {
459 swap_tagcache_header(&buf->tch);
460 buf->serial = swap32(buf->serial);
461 buf->commitid = swap32(buf->commitid);
462 buf->dirty = swap32(buf->dirty);
463 }
464}
465#else
466static void swap_tagfile_entry(struct tagfile_entry *buf) { (void)buf; }
467static void swap_index_entry(struct index_entry *buf) { (void)buf; }
468static void swap_tagcache_header(struct tagcache_header *buf) { (void)buf; }
469static void swap_master_header(struct master_header *buf) { (void)buf; }
470#endif
471
472static ssize_t read_tagfile_entry(int fd, struct tagfile_entry *buf)
473{
474 ssize_t ret = read(fd, buf, sizeof(*buf));
475 if (ret == sizeof(*buf) && tc_stat.econ)
476 swap_tagfile_entry(buf);
477
478 return ret;
479}
480
481static ssize_t write_tagfile_entry(int fd, struct tagfile_entry *buf)
482{
483 struct tagfile_entry e = *buf;
484
485 swap_tagfile_entry(&e);
486
487 return write(fd, &e, sizeof(e));
488}
489
490enum e_read_errors {
491 e_SUCCESS = 0,
492 e_SUCCESS_LEN_ZERO = 1,
493 e_ENTRY_SIZEMISMATCH,
494 e_TAG_TOOLONG,
495 e_TAG_SIZEMISMATCH
496};
497
498static enum e_read_errors
499read_tagfile_entry_and_tag(int fd, struct tagfile_entry *tfe,
500 char* buf, int bufsz)
501{
502 if (read_tagfile_entry(fd, tfe) != sizeof(struct tagfile_entry))
503 return e_ENTRY_SIZEMISMATCH;
504
505 long tag_length = tfe->tag_length;
506 if (tag_length >= bufsz)
507 return e_TAG_TOOLONG;
508
509 if (tag_length > 0 && read(fd, buf, tag_length) != tag_length)
510 return e_TAG_SIZEMISMATCH;
511
512 str_setlen(buf, tag_length);
513 return (tag_length > 0 && *buf) ? e_SUCCESS : e_SUCCESS_LEN_ZERO;
514}
515
516static ssize_t read_index_entries(int fd, struct index_entry *buf, size_t count)
517{
518 ssize_t ret = read(fd, buf, sizeof(*buf) * count);
519 for (ssize_t i = 0; i < ret; i += sizeof(*buf))
520 swap_index_entry(buf++);
521
522 return ret;
523}
524
525static ssize_t write_index_entries(int fd, struct index_entry *buf, size_t count)
526{
527#ifdef TAGCACHE_SUPPORT_FOREIGN_ENDIAN
528 ssize_t ret = 0;
529 for (; count > 0; count--)
530 {
531 struct index_entry e = *buf++;
532 swap_index_entry(&e);
533
534 ssize_t rc = write(fd, &e, sizeof(e));
535 if (rc < 0)
536 return rc;
537 ret += rc;
538 }
539
540 return ret;
541#else
542 return write(fd, buf, sizeof(*buf) * count);
543#endif
544}
545
546static ssize_t read_tagcache_header(int fd, struct tagcache_header *buf)
547{
548 ssize_t ret = read(fd, buf, sizeof(*buf));
549 if (ret == sizeof(*buf))
550 swap_tagcache_header(buf);
551
552 return ret;
553}
554
555static ssize_t write_tagcache_header(int fd, struct tagcache_header *buf)
556{
557 struct tagcache_header e = *buf;
558 swap_tagcache_header(&e);
559 return write(fd, &e, sizeof(e));
560}
561
562static ssize_t read_master_header(int fd, struct master_header *buf)
563{
564 ssize_t ret = read(fd, buf, sizeof(*buf));
565 if (ret == sizeof(*buf))
566 swap_master_header(buf);
567
568 return ret;
569}
570
571static ssize_t write_master_header(int fd, struct master_header *buf)
572{
573 struct master_header e = *buf;
574 swap_master_header(&e);
575 return write(fd, &e, sizeof(e));
576}
577
578/*
579 * open_db_fd and remove_db_file are noinline to minimize stack usage
580 */
581static int NO_INLINE open_db_fd(const char* filename, int mode)
582{
583 char buf[MAX_PATH];
584
585 if(mode & O_CREAT)
586 {
587 if (mkdir(tc_stat.db_path) < 0 && errno != EEXIST)
588 return -1;
589 }
590
591 return open_pathfmt(buf, sizeof(buf), mode, "%s/%s",
592 tc_stat.db_path, filename);
593}
594
595static int NO_INLINE remove_db_file(const char* filename)
596{
597 char buf[MAX_PATH];
598
599 snprintf(buf, sizeof(buf), "%s/%s",
600 tc_stat.db_path, filename);
601
602 return remove(buf);
603}
604
605static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
606{
607 int fd;
608 char fname[MAX_PATH];
609
610 if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
611 return -1;
612
613 fd = open_pathfmt(fname, sizeof(fname),
614 write ? O_RDWR : O_RDONLY, "%s/" TAGCACHE_FILE_INDEX,
615 tc_stat.db_path, tag);
616 if (fd < 0)
617 {
618 logf("%s failed: tag=%d write=%d file= " TAGCACHE_FILE_INDEX,
619 __func__, tag, write, tag);
620 tc_stat.ready = false;
621 return fd;
622 }
623
624 /* Check the header. */
625 if (read_tagcache_header(fd, hdr) != sizeof(struct tagcache_header) ||
626 hdr->magic != TAGCACHE_MAGIC)
627 {
628 logf("header error");
629 tc_stat.ready = false;
630 close(fd);
631 return -2;
632 }
633
634 return fd;
635}
636
637static int open_master_fd(struct master_header *hdr, bool write)
638{
639 int fd;
640 int rc;
641
642 fd = open_db_fd(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
643 if (fd < 0)
644 {
645 logf("master file open failed for R/W");
646 tc_stat.ready = false;
647 return fd;
648 }
649
650 rc = read(fd, hdr, sizeof(struct master_header));
651 if (rc != sizeof(struct master_header))
652 {
653 logf("master file read failed");
654 close(fd);
655 return -1;
656 }
657
658 /* Tagcache files can have either endianness. A device will always
659 * create files in its native endianness, but we accept non-native
660 * endian files for compatibility reasons. */
661 if (hdr->tch.magic == TAGCACHE_MAGIC)
662 tc_stat.econ = false;
663#ifdef TAGCACHE_SUPPORT_FOREIGN_ENDIAN
664 else if (hdr->tch.magic == swap32(TAGCACHE_MAGIC))
665 {
666 tc_stat.econ = true;
667 swap_master_header(hdr);
668 }
669#endif
670 else
671 {
672 logf("master file bad magic: %08lx\n", (unsigned long)hdr->tch.magic);
673 close(fd);
674 return -2;
675 }
676
677 return fd;
678}
679
680static void remove_files(void)
681{
682 int i;
683 char buf[MAX_PATH];
684 const int bufsz = sizeof(buf);
685 logf("%s", __func__);
686
687 tc_stat.ready = false;
688 tc_stat.ramcache = false;
689 tc_stat.econ = false;
690 remove_db_file(TAGCACHE_FILE_MASTER);
691 for (i = 0; i < TAG_COUNT; i++)
692 {
693 if (TAGCACHE_IS_NUMERIC(i))
694 continue;
695
696 snprintf(buf, bufsz, "%s/" TAGCACHE_FILE_INDEX,
697 tc_stat.db_path, i);
698 remove(buf);
699 }
700}
701
702static bool check_all_headers(void)
703{
704 struct master_header myhdr;
705 struct tagcache_header tch;
706 int tag;
707 int fd;
708
709 if ( (fd = open_master_fd(&myhdr, false)) < 0)
710 return false;
711
712 close(fd);
713 if (myhdr.dirty)
714 {
715 logf("tagcache is dirty!");
716 return false;
717 }
718
719 memcpy(¤t_tcmh, &myhdr, sizeof(struct master_header));
720
721 for (tag = 0; tag < TAG_COUNT; tag++)
722 {
723 if (TAGCACHE_IS_NUMERIC(tag))
724 continue;
725
726 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
727 return false;
728
729 close(fd);
730 }
731
732 return true;
733}
734
735static bool update_master_header(void)
736{
737 struct master_header myhdr;
738 int fd;
739
740 if (!tc_stat.ready)
741 return false;
742
743 if ( (fd = open_master_fd(&myhdr, true)) < 0)
744 return false;
745
746 myhdr.serial = current_tcmh.serial;
747 myhdr.commitid = current_tcmh.commitid;
748 myhdr.dirty = current_tcmh.dirty;
749
750 /* Write it back */
751 lseek(fd, 0, SEEK_SET);
752 write_master_header(fd, &myhdr);
753 close(fd);
754
755 return true;
756}
757
758#if !defined(PLUGIN)
759#ifndef __PCTOOL__
760static bool do_timed_yield(void)
761{
762 /* Sorting can lock up for quite a while, so yield occasionally */
763 static long wakeup_tick = 0;
764 if (TIME_AFTER(current_tick, wakeup_tick))
765 {
766 yield();
767 wakeup_tick = current_tick + (HZ/25);
768 return true;
769 }
770 return false;
771}
772#endif /* __PCTOOL__ */
773
774static void allocate_tempbuf(void)
775{
776 /* Yeah, malloc would be really nice now :) */
777 size_t size;
778 tempbuf_size = 0;
779
780#ifdef __PCTOOL__
781 size = 32*1024*1024;
782 tempbuf = malloc(size);
783 if (tempbuf)
784 tempbuf_size = size;
785#else /* !__PCTOOL__ */
786 /* Need to pass dummy ops to prevent the buffer being moved
787 * out from under us, since we yield during the tagcache commit. */
788 tempbuf_handle = core_alloc_maximum(&size, &buflib_ops_locked);
789 if (tempbuf_handle > 0)
790 {
791 tempbuf = core_get_data(tempbuf_handle);
792 tempbuf_size = size;
793 }
794#endif /* __PCTOOL__ */
795
796}
797
798static void free_tempbuf(void)
799{
800 if (tempbuf_size == 0)
801 return ;
802
803#ifdef __PCTOOL__
804 free(tempbuf);
805#else
806 tempbuf_handle = core_free(tempbuf_handle);
807#endif
808 tempbuf = NULL;
809 tempbuf_size = 0;
810}
811
812#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
813/* find the ramcache entry corresponding to the file indicated by
814 * filename and dc (it's corresponding dircache id). */
815static long find_entry_ram(const char *filename)
816{
817 static long last_pos = 0;
818 struct dircache_fileref dcfref;
819
820 /* Check if tagcache is loaded into ram. */
821 if (!tc_stat.ramcache
822 || global_settings.tagcache_ram != TAGCACHE_RAM_ON)
823 return -1;
824
825 if (dircache_search(DCS_CACHED_PATH | DCS_UPDATE_FILEREF, &dcfref,
826 filename) <= 0)
827 {
828 logf("tagcache: file not found.");
829 return -1;
830 }
831
832 /* Search references */
833 int end_pos = current_tcmh.tch.entry_count;
834 while (1)
835 {
836 for (int i = last_pos; i < end_pos; i++)
837 {
838 do_timed_yield();
839
840 if (!(tcramcache.hdr->indices[i].flag & FLAG_DIRCACHE))
841 continue;
842
843 int cmp = dircache_fileref_cmp(&tcrc_dcfrefs[i], &dcfref);
844 if (cmp < 3)
845 continue;
846
847 last_pos = MAX(0, i - 3);
848 return i;
849 }
850
851 if (last_pos == 0)
852 {
853 last_pos = MAX(0, end_pos - 3);
854 break;
855 }
856
857 end_pos = last_pos;
858 last_pos = 0;
859 }
860
861 return -1;
862}
863#endif /* defined (HAVE_TC_RAMCACHE) && defined (HAVE_DIRCACHE) */
864
865static long find_entry_disk(const char *filename_raw, bool localfd)
866{
867 struct tagfile_entry tfe;
868 struct tagcache_header tch;
869 static long last_pos = -1;
870 long pos_history[POS_HISTORY_COUNT];
871 unsigned int pos_history_idx = 0;
872 unsigned int i;
873
874 char buf[TAGCACHE_BUFSZ];
875 const long bufsz = sizeof(buf);
876
877 int fd;
878 int pos = -1;
879 long idx = -1;
880
881 bool found = false;
882
883 const char *filename = filename_raw;
884
885#ifdef APPLICATION
886 char pathbuf[PATH_MAX]; /* Note: Don't use MAX_PATH here, it's too small */
887 if (realpath(filename, pathbuf) == pathbuf)
888 filename = pathbuf;
889#endif /* APPLICATION */
890
891 if (!tc_stat.ready)
892 return -2;
893
894 fd = filenametag_fd;
895 if (fd < 0 || localfd)
896 {
897 last_pos = -1;
898 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
899 return -1;
900 }
901
902 check_again:
903
904 if (last_pos > 0) /* pos gets cached to prevent reading from beginning */
905 pos = lseek(fd, last_pos, SEEK_SET);
906 else /* start back at beginning */
907 pos = lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
908
909 long tag_length = strlen(filename) + 1; /* include NULL */
910
911 if (tag_length < bufsz)
912 {
913 while (true)
914 {
915 for (i = pos_history_idx-1; i < pos_history_idx; i--)
916 pos_history[i+1] = pos_history[i];
917 pos_history[0] = pos;
918
919 if (read_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
920 {
921 logf("size mismatch find entry");
922 break;
923 }
924 else
925 {
926 pos += sizeof(struct tagfile_entry) + tfe.tag_length;
927 /* don't read the entry unless the length matches */
928 if (tfe.tag_length == tag_length)
929 {
930 if(read(fd, buf, tfe.tag_length) != tag_length)
931 {
932 logf("read error #2");
933 close(fd);
934 if (!localfd)
935 filenametag_fd = -1;
936 last_pos = -1;
937 return -3;
938 }
939 if (!strncmp(filename, buf, tag_length))
940 {
941 last_pos = pos_history[pos_history_idx];
942 found = true;
943 idx = tfe.idx_id;
944 break ;
945 }
946 }
947 else
948 lseek(fd, pos, SEEK_SET);
949
950 }
951 }
952 if (pos_history_idx < POS_HISTORY_COUNT - 1)
953 pos_history_idx++;
954 }
955
956 /* Not found? */
957 if (!found)
958 {
959 if (last_pos > 0) /* start back at the beginning */
960 {
961 last_pos = -1;
962 logf("seek again");
963 goto check_again;
964 }
965
966 idx = -4;
967 }
968
969 if (fd != filenametag_fd || localfd)
970 close(fd);
971
972 return idx;
973}
974
975static int find_index(const char *filename)
976{
977 long idx_id = -1;
978
979#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
980 idx_id = find_entry_ram(filename);
981#endif
982
983 if (idx_id < 0)
984 idx_id = find_entry_disk(filename, true);
985
986 return idx_id;
987}
988
989bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
990{
991 /* NOTE: on ret==true you need to call tagcache_search_finish(tcs) yourself */
992 int idx_id;
993
994 if (!tc_stat.ready)
995 return false;
996
997 idx_id = find_index(filename);
998 if (idx_id < 0)
999 return false;
1000
1001 if (!tagcache_search(tcs, tag_filename))
1002 return false;
1003
1004 tcs->entry_count = 0;
1005 tcs->idx_id = idx_id;
1006
1007 return true;
1008}
1009
1010static bool get_index(int masterfd, int idxid,
1011 struct index_entry *idx, bool use_ram)
1012{
1013 bool localfd = false;
1014
1015 if (idxid < 0)
1016 {
1017 logf("Incorrect idxid: %d", idxid);
1018 return false;
1019 }
1020
1021#ifdef HAVE_TC_RAMCACHE
1022 if (tc_stat.ramcache && use_ram)
1023 {
1024 if (tcramcache.hdr->indices[idxid].flag & FLAG_DELETED)
1025 return false;
1026
1027 *idx = tcramcache.hdr->indices[idxid];
1028 return true;
1029 }
1030#endif /* HAVE_TC_RAMCACHE */
1031
1032 if (masterfd < 0)
1033 {
1034 struct master_header tcmh;
1035
1036 localfd = true;
1037 masterfd = open_master_fd(&tcmh, false);
1038 if (masterfd < 0)
1039 return false;
1040 }
1041
1042 lseek(masterfd, idxid * sizeof(struct index_entry)
1043 + sizeof(struct master_header), SEEK_SET);
1044 if (read_index_entries(masterfd, idx, 1) != sizeof(struct index_entry))
1045 {
1046 logf("read error #3");
1047 if (localfd)
1048 close(masterfd);
1049
1050 return false;
1051 }
1052
1053 if (localfd)
1054 close(masterfd);
1055
1056 if (idx->flag & FLAG_DELETED)
1057 return false;
1058
1059 return true;
1060
1061 (void)use_ram;
1062}
1063
1064#ifndef __PCTOOL__
1065
1066static bool write_index(int masterfd, int idxid, struct index_entry *idx)
1067{
1068 /* We need to exclude all memory only flags & tags when writing to disk. */
1069 if (idx->flag & FLAG_DIRCACHE)
1070 {
1071 logf("memory only flags!");
1072 return false;
1073 }
1074
1075#ifdef HAVE_TC_RAMCACHE
1076 /* Only update numeric data. Writing the whole index to RAM by memcpy
1077 * destroys dircache pointers!
1078 */
1079 if (tc_stat.ramcache)
1080 {
1081 struct index_entry *idx_ram = &tcramcache.hdr->indices[idxid];
1082
1083 for (int tag = 0; tag < TAG_COUNT; tag++)
1084 {
1085 if (TAGCACHE_IS_NUMERIC(tag))
1086 {
1087 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
1088 }
1089 }
1090
1091 /* Don't touch the dircache flag or attributes. */
1092 idx_ram->flag = (idx->flag & 0x0000ffff)
1093 | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
1094 }
1095#endif /* HAVE_TC_RAMCACHE */
1096
1097 lseek(masterfd, idxid * sizeof(struct index_entry)
1098 + sizeof(struct master_header), SEEK_SET);
1099 if (write_index_entries(masterfd, idx, 1) != sizeof(struct index_entry))
1100 {
1101 logf("write error #3");
1102 logf("idxid: %d", idxid);
1103 return false;
1104 }
1105
1106 return true;
1107}
1108
1109#endif /* !__PCTOOL__ */
1110
1111static bool open_files(struct tagcache_search *tcs, int tag)
1112{
1113 if (tcs->idxfd[tag] < 0)
1114 {
1115 char fname[MAX_PATH];
1116 tcs->idxfd[tag] = open_pathfmt(fname, sizeof(fname),
1117 O_RDONLY, "%s/" TAGCACHE_FILE_INDEX,
1118 tc_stat.db_path, tag);
1119 if (tcs->idxfd[tag] < 0)
1120 {
1121 logf("File not open!");
1122 return false;
1123 }
1124 }
1125
1126 return true;
1127}
1128
1129static bool retrieve(struct tagcache_search *tcs, IF_DIRCACHE(int idx_id,)
1130 struct index_entry *idx, int tag, char *buf, long bufsz)
1131{
1132 bool success = false;
1133 bool is_basename = false;
1134 struct tagfile_entry tfe;
1135 long seek;
1136
1137 if (tag == tag_virt_basename)
1138 {
1139 tag = tag_filename;
1140 is_basename = true;
1141 }
1142
1143 if (TAGCACHE_IS_NUMERIC(tag))
1144 goto failure;
1145
1146 seek = idx->tag_seek[tag];
1147 if (seek < 0)
1148 {
1149 logf("Retrieve failed");
1150 goto failure;
1151 }
1152
1153#ifdef HAVE_TC_RAMCACHE
1154 if (tcs->ramsearch)
1155 {
1156#ifdef HAVE_DIRCACHE
1157 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE))
1158 {
1159 if (dircache_get_fileref_path(&tcrc_dcfrefs[idx_id], buf, bufsz) >= 0)
1160 success = true;
1161 }
1162 else
1163#endif /* HAVE_DIRCACHE */
1164 if (tag != tag_filename)
1165 {
1166 struct tagfile_entry *ep =
1167 (struct tagfile_entry *)&tcramcache.hdr->tags[tag][seek];
1168 strmemccpy(buf, ep->tag_data, bufsz);
1169 success = true;
1170 }
1171 }
1172#endif /* HAVE_TC_RAMCACHE */
1173
1174 if (!success && open_files(tcs, tag))
1175 {
1176 lseek(tcs->idxfd[tag], seek, SEEK_SET);
1177 switch (read_tagfile_entry_and_tag(tcs->idxfd[tag], &tfe, buf, bufsz))
1178 {
1179 case e_ENTRY_SIZEMISMATCH:
1180 logf("read error #5");
1181 break;
1182 case e_TAG_TOOLONG:
1183 logf("too long tag #5");
1184 break;
1185 case e_TAG_SIZEMISMATCH:
1186 logf("read error #6");
1187 break;
1188 default:
1189 logf("unknown_error");
1190 break;
1191 case e_SUCCESS_LEN_ZERO:
1192 case e_SUCCESS:
1193 success = true;
1194 break;
1195 }
1196 }
1197
1198 if (success)
1199 {
1200 if (is_basename)
1201 {
1202 char* basename = strrchr(buf, '/');
1203 if (basename != NULL)
1204 memmove(buf, basename + 1, strlen(basename)); /* includes NULL */
1205 }
1206 return true;
1207 }
1208
1209failure:
1210 str_setlen(buf, 0);
1211 return false;
1212}
1213
1214#define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
1215
1216static long tc_find_tag(int tag, int idx_id, const struct index_entry *idx)
1217{
1218#ifndef __PCTOOL__
1219 if (! COMMAND_QUEUE_IS_EMPTY && TAGCACHE_IS_NUMERIC(tag))
1220 {
1221 /* Attempt to find tag data through store-to-load forwarding in
1222 command queue */
1223 long result = -1;
1224
1225 mutex_lock(&command_queue_mutex);
1226
1227 int ridx = command_queue_widx;
1228
1229 while (ridx != command_queue_ridx)
1230 {
1231 if (--ridx < 0)
1232 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
1233
1234 if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
1235 && command_queue[ridx].idx_id == idx_id
1236 && command_queue[ridx].tag == tag)
1237 {
1238 result = command_queue[ridx].data;
1239 break;
1240 }
1241 }
1242
1243 mutex_unlock(&command_queue_mutex);
1244
1245 if (result >= 0)
1246 {
1247 logf("tc_find_tag: Recovered tag %d value %lX from write queue",
1248 tag, (unsigned long) result);
1249 return result;
1250 }
1251 }
1252#else
1253 (void)idx_id;
1254#endif
1255
1256 return idx->tag_seek[tag];
1257}
1258
1259static inline long sec_in_ms(long ms)
1260{
1261 return (ms/1000) % 60;
1262}
1263
1264static inline long min_in_ms(long ms)
1265{
1266 return (ms/1000) / 60;
1267}
1268
1269static long check_virtual_tags(int tag, int idx_id,
1270 const struct index_entry *idx)
1271{
1272 long data = 0;
1273
1274 switch (tag)
1275 {
1276 case tag_virt_length_sec:
1277 data = sec_in_ms(tc_find_tag(tag_length, idx_id, idx));
1278 break;
1279
1280 case tag_virt_length_min:
1281 data = min_in_ms(tc_find_tag(tag_length, idx_id, idx));
1282 break;
1283
1284 case tag_virt_playtime_sec:
1285 data = sec_in_ms(tc_find_tag(tag_playtime, idx_id, idx));
1286 break;
1287
1288 case tag_virt_playtime_min:
1289 data = min_in_ms(tc_find_tag(tag_playtime, idx_id, idx));
1290 break;
1291
1292 case tag_virt_autoscore:
1293 if (tc_find_tag(tag_length, idx_id, idx) == 0
1294 || tc_find_tag(tag_playcount, idx_id, idx) == 0)
1295 {
1296 data = 0;
1297 }
1298 else
1299 {
1300 /* A straight calculus gives:
1301 autoscore = 100 * playtime / length / playcout (1)
1302 Now, consider the euclidian division of playtime by length:
1303 playtime = alpha * length + beta
1304 With:
1305 0 <= beta < length
1306 Now, (1) becomes:
1307 autoscore = 100 * (alpha / playcout + beta / length / playcount)
1308 Both terms should be small enough to avoid any overflow
1309 */
1310 long playtime = tc_find_tag(tag_playtime, idx_id, idx);
1311 long length = tc_find_tag(tag_length, idx_id, idx);
1312 long playcount = tc_find_tag(tag_playcount, idx_id, idx);
1313 data = 100 * (playtime / length) + (100 * (playtime % length)) / length;
1314 data /= playcount;
1315 }
1316 break;
1317
1318 /* How many commits before the file has been added to the DB. */
1319 case tag_virt_entryage:
1320 data = current_tcmh.commitid
1321 - tc_find_tag(tag_commitid, idx_id, idx) - 1;
1322 break;
1323
1324 case tag_virt_basename:
1325 tag = tag_filename; /* return filename; caller handles basename */
1326 /* FALLTHRU */
1327
1328 default:
1329 data = tc_find_tag(tag, idx_id, idx);
1330 }
1331
1332 return data;
1333}
1334
1335long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
1336{
1337 struct index_entry idx;
1338
1339 if (!tc_stat.ready)
1340 return false;
1341
1342 if (!TAGCACHE_IS_NUMERIC(tag))
1343 return -1;
1344
1345 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1346 return -2;
1347
1348 return check_virtual_tags(tag, tcs->idx_id, &idx);
1349}
1350
1351inline static bool str_ends_with(const char *str1, const char *str2)
1352{
1353 logf_clauses("%s %s %s", str1, __func__, str2);
1354 int str_len = strlen(str1);
1355 int clause_len = strlen(str2);
1356
1357 if (clause_len > str_len)
1358 return false;
1359
1360 return !strcasecmp(&str1[str_len - clause_len], str2);
1361}
1362
1363inline static bool str_oneof(const char *str, const char *list)
1364{
1365 logf_clauses("%s %s %s", str, __func__, list);
1366 const char *sep;
1367 int l, len = strlen(str);
1368
1369 while (*list)
1370 {
1371 sep = strchr(list, '|');
1372 l = sep ? (intptr_t)sep - (intptr_t)list : (int)strlen(list);
1373 if ((l==len) && !strncasecmp(str, list, len))
1374 return true;
1375 list += sep ? l + 1 : l;
1376 }
1377
1378 return false;
1379}
1380
1381inline static bool str_begins_ends_oneof(const char *str, const char *list, bool begins)
1382{
1383 logf_clauses("%s %s (%s) %s", str, __func__, begins ? "begins" : "ends", list);
1384 const char *sep;
1385 int l, p, len = strlen(str);
1386
1387 while (*list)
1388 {
1389 sep = strchr(list, '|');
1390 l = sep ? (intptr_t)sep - (intptr_t)list : (int)strlen(list);
1391 p = begins ? 0 : len - l;
1392 if (l <= len && !strncasecmp(&str[p], list, l))
1393 return true;
1394 list += sep ? l + 1 : l;
1395 }
1396
1397 return false;
1398}
1399
1400static bool check_against_clause(long numeric, const char *str,
1401 const struct tagcache_search_clause *clause)
1402{
1403 if (clause->numeric)
1404 {
1405 switch (clause->type)
1406 {
1407 case clause_is:
1408 return numeric == clause->numeric_data;
1409 case clause_is_not:
1410 return numeric != clause->numeric_data;
1411 case clause_gt:
1412 return numeric > clause->numeric_data;
1413 case clause_gteq:
1414 return numeric >= clause->numeric_data;
1415 case clause_lt:
1416 return numeric < clause->numeric_data;
1417 case clause_lteq:
1418 return numeric <= clause->numeric_data;
1419 default:
1420 logf("Incorrect numeric tag: %d", clause->type);
1421 }
1422 }
1423 else
1424 {
1425 switch (clause->type)
1426 {
1427 case clause_is:
1428 return !strcasecmp(clause->str, str);
1429 case clause_is_not:
1430 return strcasecmp(clause->str, str);
1431 case clause_gt:
1432 return 0>strcasecmp(clause->str, str);
1433 case clause_gteq:
1434 return 0>=strcasecmp(clause->str, str);
1435 case clause_lt:
1436 return 0<strcasecmp(clause->str, str);
1437 case clause_lteq:
1438 return 0<=strcasecmp(clause->str, str);
1439 case clause_contains:
1440 return (strcasestr(str, clause->str) != NULL);
1441 case clause_not_contains:
1442 return (strcasestr(str, clause->str) == NULL);
1443 case clause_begins_with:
1444 return (strcasestr(str, clause->str) == str);
1445 case clause_not_begins_with:
1446 return (strcasestr(str, clause->str) != str);
1447 case clause_ends_with:
1448 return str_ends_with(str, clause->str);
1449 case clause_not_ends_with:
1450 return !str_ends_with(str, clause->str);
1451 case clause_oneof:
1452 return str_oneof(str, clause->str);
1453 case clause_not_oneof:
1454 return !str_oneof(str, clause->str);
1455 case clause_ends_oneof:
1456 /* Fall-Through */
1457 case clause_begins_oneof:
1458 return str_begins_ends_oneof(str, clause->str,
1459 clause->type == clause_begins_oneof);
1460 case clause_not_ends_oneof:
1461 /* Fall-Through */
1462 case clause_not_begins_oneof:
1463 return !str_begins_ends_oneof(str, clause->str,
1464 clause->type == clause_not_begins_oneof);
1465 default:
1466 logf("Incorrect tag: %d", clause->type);
1467 }
1468 }
1469
1470 return false;
1471}
1472
1473static bool check_clauses(struct tagcache_search *tcs,
1474 struct index_entry *idx,
1475 struct tagcache_search_clause **clauses, int count)
1476{
1477 int i;
1478
1479 /* Go through all conditional clauses. */
1480 for (i = 0; i < count; i++)
1481 {
1482 int seek;
1483 char buf[256];
1484 const int bufsz = sizeof(buf);
1485 char *str = buf;
1486 struct tagcache_search_clause *clause = clauses[i];
1487
1488 logf_clauses("%s clause %d %s %s [%ld] %s",
1489 "Checking", i, tag_type_str[clause->type],
1490 tags_str[clause->tag], clause->numeric_data,
1491 (clause->numeric || clause->str == NULL) ? "[NUMERIC?]" : clause->str);
1492
1493 if (clause->type == clause_logical_or)
1494 {
1495 logf_clauses("Bailing");
1496 break; /* all conditions before logical-or satisfied --
1497 stop processing clauses */
1498 }
1499 seek = check_virtual_tags(clause->tag, tcs->idx_id, idx);
1500
1501#ifdef HAVE_TC_RAMCACHE
1502 if (tcs->ramsearch)
1503 {
1504 struct tagfile_entry *tfe;
1505
1506 if (!TAGCACHE_IS_NUMERIC(clause->tag))
1507 {
1508 if (clause->tag == tag_filename
1509 || clause->tag == tag_virt_basename)
1510 {
1511 retrieve(tcs, IF_DIRCACHE(tcs->idx_id,) idx, clause->tag,
1512 buf, bufsz);
1513 }
1514 else
1515 {
1516 tfe = (struct tagfile_entry *)
1517 &tcramcache.hdr->tags[clause->tag][seek];
1518 /* str points to movable data, but no locking required here,
1519 * as no yield() is following */
1520 str = tfe->tag_data;
1521 }
1522 }
1523 }
1524 else
1525#endif /* HAVE_TC_RAMCACHE */
1526 {
1527 struct tagfile_entry tfe;
1528
1529 if (!TAGCACHE_IS_NUMERIC(clause->tag))
1530 {
1531 int tag = clause->tag;
1532 if (tag == tag_virt_basename)
1533 tag = tag_filename;
1534
1535 int fd = tcs->idxfd[tag];
1536 lseek(fd, seek, SEEK_SET);
1537
1538 switch (read_tagfile_entry_and_tag(fd, &tfe, str, bufsz))
1539 {
1540 case e_SUCCESS_LEN_ZERO: /* Check if entry has been deleted. */
1541 return false;
1542 case e_SUCCESS:
1543 if (clause->tag == tag_virt_basename)
1544 {
1545 char *basename = strrchr(str, '/');
1546 if (basename)
1547 str = basename + 1;
1548 }
1549 break;
1550 case e_ENTRY_SIZEMISMATCH:
1551 logf("read error #15");
1552 return false;
1553 case e_TAG_TOOLONG:
1554 logf("too long tag #6");
1555 return false;
1556 case e_TAG_SIZEMISMATCH:
1557 logf("read error #16");
1558 return false;
1559 default:
1560 logf("unknown_error");
1561 break;;
1562 }
1563 }
1564 }
1565
1566 if (!check_against_clause(seek, str, clause))
1567 {
1568 /* Clause failed -- try finding a logical-or clause */
1569 while (++i < count)
1570 {
1571 if (clauses[i]->type == clause_logical_or)
1572 break;
1573 }
1574
1575 if (i < count) /* Found logical-or? */
1576 continue; /* Check clauses after logical-or */
1577
1578 return false;
1579 }
1580
1581 logf_clauses("%s clause %d %s %s [%ld] %s",
1582 "Found", i, tag_type_str[clause->type],
1583 tags_str[clause->tag], clause->numeric_data,
1584 (clause->numeric || clause->str == NULL) ? "[NUMERIC?]" : clause->str);
1585 }
1586
1587 return true;
1588}
1589
1590bool tagcache_check_clauses(struct tagcache_search *tcs,
1591 struct tagcache_search_clause **clause, int count)
1592{
1593 struct index_entry idx;
1594
1595 if (count == 0)
1596 return true;
1597
1598 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1599 return false;
1600
1601 return check_clauses(tcs, &idx, clause, count);
1602}
1603
1604static bool add_uniqbuf(struct tagcache_search *tcs, uint32_t id)
1605{
1606 int i;
1607
1608 /* If uniq buffer is not defined we must return true for search to work. */
1609 if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1610 && !TAGCACHE_IS_NUMERIC(tcs->type)))
1611 {
1612 return true;
1613 }
1614
1615 for (i = 0; i < tcs->unique_list_count; i++)
1616 {
1617 /* Return false if entry is found. */
1618 if (tcs->unique_list[i] == id)
1619 {
1620 //logf("%d Exists @ %d", id, i);
1621 return false;
1622 }
1623 }
1624
1625 if (tcs->unique_list_count < tcs->unique_list_capacity)
1626 {
1627 tcs->unique_list[i] = id;
1628 tcs->unique_list_count++;
1629 }
1630
1631 return true;
1632}
1633
1634static bool build_lookup_list(struct tagcache_search *tcs)
1635{
1636 struct index_entry entry;
1637 int i, j;
1638
1639 tcs->seek_list_count = 0;
1640
1641#ifdef HAVE_TC_RAMCACHE
1642 if (tcs->ramsearch)
1643 {
1644 tcrc_buffer_lock(); /* lock because below makes a pointer to movable data */
1645
1646 for (i = tcs->seek_pos; i < current_tcmh.tch.entry_count; i++)
1647 {
1648 struct tagcache_seeklist_entry *seeklist;
1649 /* idx points to movable data, don't yield or reload */
1650 struct index_entry *idx = &tcramcache.hdr->indices[i];
1651 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1652 break ;
1653
1654 /* Skip deleted files. */
1655 if (idx->flag & FLAG_DELETED)
1656 continue;
1657
1658 /* Go through all filters.. */
1659 for (j = 0; j < tcs->filter_count; j++)
1660 {
1661 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1662 {
1663 break ;
1664 }
1665 }
1666
1667 if (j < tcs->filter_count)
1668 continue ;
1669
1670 /* Check for conditions. */
1671 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1672 continue;
1673 /* Add to the seek list if not already in uniq buffer (doesn't yield)*/
1674 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1675 continue;
1676
1677 /* Lets add it. */
1678 seeklist = &tcs->seeklist[tcs->seek_list_count];
1679 seeklist->seek = idx->tag_seek[tcs->type];
1680 seeklist->flag = idx->flag;
1681 seeklist->idx_id = i;
1682 tcs->seek_list_count++;
1683 }
1684
1685 tcrc_buffer_unlock();
1686
1687 tcs->seek_pos = i;
1688
1689 return tcs->seek_list_count > 0;
1690 }
1691#endif /* HAVE_TC_RAMCACHE */
1692
1693 if (tcs->masterfd < 0)
1694 {
1695 struct master_header tcmh;
1696 tcs->masterfd = open_master_fd(&tcmh, false);
1697 }
1698
1699 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1700 sizeof(struct master_header), SEEK_SET);
1701
1702 while (read_index_entries(tcs->masterfd, &entry, 1) == sizeof(struct index_entry))
1703 {
1704 struct tagcache_seeklist_entry *seeklist;
1705
1706 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1707 break ;
1708
1709 i = tcs->seek_pos;
1710 tcs->seek_pos++;
1711
1712 /* Check if entry has been deleted. */
1713 if (entry.flag & FLAG_DELETED)
1714 continue;
1715
1716 /* Go through all filters.. */
1717 for (j = 0; j < tcs->filter_count; j++)
1718 {
1719 if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1720 break ;
1721 }
1722
1723 if (j < tcs->filter_count)
1724 continue ;
1725
1726 /* Check for conditions. */
1727 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1728 continue;
1729
1730 /* Add to the seek list if not already in uniq buffer. */
1731 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1732 continue;
1733
1734 /* Lets add it. */
1735 seeklist = &tcs->seeklist[tcs->seek_list_count];
1736 seeklist->seek = entry.tag_seek[tcs->type];
1737 seeklist->flag = entry.flag;
1738 seeklist->idx_id = i;
1739 tcs->seek_list_count++;
1740
1741 yield();
1742 }
1743
1744 return tcs->seek_list_count > 0;
1745}
1746
1747
1748bool tagcache_search(struct tagcache_search *tcs, int tag)
1749{
1750 /* NOTE: call tagcache_search_finish(&tcs) when finished or BAD things may happen (TM) */
1751 struct tagcache_header tag_hdr;
1752 struct master_header master_hdr;
1753 int i;
1754
1755 while (read_lock)
1756 sleep(1);
1757
1758 memset(tcs, 0, sizeof(struct tagcache_search));
1759 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1760 return false;
1761
1762 tcs->position = sizeof(struct tagcache_header);
1763 tcs->type = tag;
1764 tcs->seek_pos = 0;
1765 tcs->list_position = 0;
1766 tcs->seek_list_count = 0;
1767 tcs->filter_count = 0;
1768 tcs->masterfd = -1;
1769
1770 for (i = 0; i < TAG_COUNT; i++)
1771 tcs->idxfd[i] = -1;
1772
1773#ifndef HAVE_TC_RAMCACHE
1774 tcs->ramsearch = false;
1775#else
1776 tcs->ramsearch = tc_stat.ramcache;
1777 if (tcs->ramsearch)
1778 {
1779 tcs->entry_count = tcramcache.hdr->entry_count[tcs->type];
1780 }
1781 else
1782#endif
1783 {
1784 /* Always open as R/W so we can pass tcs to functions that modify data also
1785 * without failing. */
1786 tcs->masterfd = open_master_fd(&master_hdr, true);
1787 if (tcs->masterfd < 0)
1788 return false;
1789
1790 if (!TAGCACHE_IS_NUMERIC(tcs->type))
1791 {
1792 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1793 if (tcs->idxfd[tcs->type] < 0)
1794 return false;
1795
1796 tcs->entry_count = tag_hdr.entry_count;
1797 }
1798 else
1799 {
1800 tcs->entry_count = master_hdr.tch.entry_count;
1801 }
1802 }
1803
1804 tcs->valid = true;
1805 tcs->initialized = true;
1806 write_lock++;
1807
1808 return true;
1809}
1810
1811void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1812 void *buffer, long length)
1813{
1814 tcs->unique_list = (uint32_t *)buffer;
1815 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1816 tcs->unique_list_count = 0;
1817 memset(tcs->unique_list, 0, tcs->unique_list_capacity);
1818}
1819
1820bool tagcache_search_add_filter(struct tagcache_search *tcs,
1821 int tag, int seek)
1822{
1823 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1824 return false;
1825
1826 if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1827 return false;
1828
1829 tcs->filter_tag[tcs->filter_count] = tag;
1830 tcs->filter_seek[tcs->filter_count] = seek;
1831 tcs->filter_count++;
1832
1833 return true;
1834}
1835
1836bool tagcache_search_add_clause(struct tagcache_search *tcs,
1837 struct tagcache_search_clause *clause)
1838{
1839 int i;
1840 int clause_count = tcs->clause_count;
1841
1842 if (clause_count >= TAGCACHE_MAX_CLAUSES)
1843 {
1844 logf("Too many clauses");
1845 return false;
1846 }
1847
1848 if (clause->type != clause_logical_or)
1849 {
1850 /* BUGFIX OR'd clauses seem to be mishandled once made into a filter */
1851 if (clause_count <= 1 || tcs->clause[clause_count - 1]->type != clause_logical_or)
1852 {
1853 /* Check if there is already a similar filter in present (filters are
1854 * much faster than clauses).
1855 */
1856 for (i = 0; i < tcs->filter_count; i++)
1857 {
1858 if (tcs->filter_tag[i] == clause->tag)
1859 {
1860 return true;
1861 }
1862 }
1863 }
1864
1865 if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1866 {
1867 char fname[MAX_PATH];
1868 tcs->idxfd[clause->tag] = open_pathfmt(fname, sizeof(fname), O_RDONLY,
1869 "%s/" TAGCACHE_FILE_INDEX,
1870 tc_stat.db_path, clause->tag);
1871 }
1872 }
1873
1874 tcs->clause[tcs->clause_count] = clause;
1875 tcs->clause_count++;
1876
1877 return true;
1878}
1879
1880static bool get_next(struct tagcache_search *tcs, bool is_numeric, char *buf, long bufsz)
1881{
1882 struct tagfile_entry entry;
1883#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1884 long flag = 0;
1885#endif
1886
1887 if (tcs->idxfd[tcs->type] < 0 && !is_numeric
1888#ifdef HAVE_TC_RAMCACHE
1889 && !tcs->ramsearch
1890#endif
1891 )
1892 return false;
1893
1894 /* Relative fetch. */
1895 if (tcs->filter_count > 0 || tcs->clause_count > 0 || is_numeric
1896#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1897 /* We need to retrieve flag status for dircache. */
1898 || (tcs->ramsearch && tcs->type == tag_filename)
1899#endif
1900 )
1901 {
1902 struct tagcache_seeklist_entry *seeklist;
1903
1904 /* Check for end of list. */
1905 if (tcs->list_position == tcs->seek_list_count)
1906 {
1907 tcs->list_position = 0;
1908
1909 /* Try to fetch more. */
1910 if (!build_lookup_list(tcs))
1911 {
1912 tcs->valid = false;
1913 return false;
1914 }
1915 }
1916
1917 seeklist = &tcs->seeklist[tcs->list_position];
1918#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1919 flag = seeklist->flag;
1920#endif
1921 tcs->position = seeklist->seek;
1922 tcs->idx_id = seeklist->idx_id;
1923 tcs->list_position++;
1924 }
1925 else
1926 {
1927 if (tcs->entry_count == 0)
1928 {
1929 tcs->valid = false;
1930 return false;
1931 }
1932
1933 tcs->entry_count--;
1934 }
1935
1936 tcs->result_seek = tcs->position;
1937
1938 if (is_numeric)
1939 {
1940 itoa_buf(buf, bufsz, tcs->position);
1941 tcs->result = buf;
1942 tcs->result_len = strlen(buf) + 1;
1943 return true;
1944 }
1945
1946 /* Direct fetch. */
1947#ifdef HAVE_TC_RAMCACHE
1948 if (tcs->ramsearch)
1949 {
1950#ifdef HAVE_DIRCACHE
1951 if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE))
1952 {
1953 ssize_t len = dircache_get_fileref_path(&tcrc_dcfrefs[tcs->idx_id],
1954 buf, bufsz);
1955 if (len >= 0)
1956 {
1957 tcs->result_len = len + 1;
1958 tcs->result = buf;
1959 tcs->ramresult = false;
1960 return true;
1961 }
1962 /* else do it the hard way */
1963 }
1964#endif /* HAVE_DIRCACHE */
1965
1966 if (tcs->type != tag_filename)
1967 {
1968 struct tagfile_entry *ep;
1969
1970 ep = (struct tagfile_entry *)&tcramcache.hdr->tags[tcs->type][tcs->position];
1971 /* don't return ep->tag_data directly as it may move */
1972 tcs->result_len = strlcpy(buf, ep->tag_data, bufsz) + 1;
1973 tcs->result = buf;
1974 tcs->idx_id = ep->idx_id;
1975 tcs->ramresult = false; /* was true before we copied to buf too */
1976
1977 /* Increase position for the next run. This may get overwritten. */
1978 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1979
1980 return true;
1981 }
1982 }
1983#endif /* HAVE_TC_RAMCACHE */
1984
1985 if (!open_files(tcs, tcs->type))
1986 {
1987 tcs->valid = false;
1988 return false;
1989 }
1990
1991 /* Seek stream to the correct position and continue to direct fetch. */
1992 lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1993
1994 switch (read_tagfile_entry_and_tag(tcs->idxfd[tcs->type], &entry, buf, bufsz))
1995 {
1996 case e_SUCCESS_LEN_ZERO:
1997 case e_SUCCESS:
1998 break;
1999 case e_ENTRY_SIZEMISMATCH:
2000 logf("read error #5");
2001 tcs->valid = false;
2002 return false;
2003 case e_TAG_TOOLONG:
2004 tcs->valid = false;
2005 logf("too long tag #2");
2006 logf("P:%lX/%" PRIX32, (unsigned long) tcs->position, entry.tag_length);
2007 return false;
2008 case e_TAG_SIZEMISMATCH:
2009 tcs->valid = false;
2010 logf("read error #4");
2011 return false;
2012 }
2013
2014 /**
2015 Update the position for the next read (this may be overridden
2016 if filters or clauses are being used).
2017 */
2018 tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
2019 str_setlen(buf, entry.tag_length);
2020
2021 tcs->result = buf;
2022 tcs->result_len = entry.tag_length + 1;
2023 tcs->idx_id = entry.idx_id;
2024 tcs->ramresult = false;
2025
2026 return true;
2027}
2028
2029bool tagcache_get_next(struct tagcache_search *tcs, char *buf, long size)
2030{
2031 if (tcs->valid && tagcache_is_usable())
2032 {
2033 bool is_numeric = TAGCACHE_IS_NUMERIC(tcs->type);
2034 while (get_next(tcs, is_numeric, buf, size))
2035 {
2036 if (tcs->result_len > 1)
2037 return true;
2038 }
2039 }
2040#ifdef LOGF_ENABLE
2041 if (tcs->unique_list_count > 0)
2042 logf(" uniqbuf: %d used / %d avail", tcs->unique_list_count, tcs->unique_list_capacity);
2043#endif
2044
2045 return false;
2046}
2047
2048bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
2049 int tag, char *buf, long size)
2050{
2051 struct index_entry idx;
2052
2053 *buf = '\0';
2054 if (!get_index(tcs->masterfd, idxid, &idx, true))
2055 return false;
2056
2057 return retrieve(tcs, IF_DIRCACHE(idxid,) &idx, tag, buf, size);
2058}
2059
2060void tagcache_search_finish(struct tagcache_search *tcs)
2061{
2062 int i;
2063
2064 if (!tcs->initialized)
2065 return;
2066
2067 if (tcs->masterfd >= 0)
2068 {
2069 close(tcs->masterfd);
2070 tcs->masterfd = -1;
2071 }
2072
2073 for (i = 0; i < TAG_COUNT; i++)
2074 {
2075 if (tcs->idxfd[i] >= 0)
2076 {
2077 close(tcs->idxfd[i]);
2078 tcs->idxfd[i] = -1;
2079 }
2080 }
2081
2082 tcs->ramsearch = false;
2083 tcs->valid = false;
2084 tcs->initialized = 0;
2085 if (write_lock > 0)
2086 write_lock--;
2087}
2088
2089#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
2090static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
2091{
2092 return (struct tagfile_entry *)&tcramcache.hdr->tags[tag][entry->tag_seek[tag]];
2093}
2094
2095static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
2096{
2097 return check_virtual_tags(tag, idx_id, entry);
2098}
2099
2100static char* get_tag_string(const struct index_entry *entry, int tag)
2101{
2102 char* s = get_tag(entry, tag)->tag_data;
2103 return strcmp(s, UNTAGGED) ? s : NULL;
2104}
2105
2106bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
2107{
2108 struct index_entry *entry;
2109 int idx_id;
2110
2111 if (!tc_stat.ready || !tc_stat.ramcache)
2112 return false;
2113
2114 /* Find the corresponding entry in tagcache. */
2115
2116 if (filename != NULL)
2117 memset(id3, 0, sizeof(struct mp3entry));
2118 else /* Note: caller clears id3 prior to call */
2119 filename = id3->path;
2120
2121 idx_id = find_entry_ram(filename);
2122 if (idx_id < 0)
2123 return false;
2124
2125 entry = &tcramcache.hdr->indices[idx_id];
2126
2127 char* buf = id3->id3v2buf;
2128 ssize_t remaining = sizeof(id3->id3v2buf);
2129
2130 /* this macro sets id3 strings by copying to the id3v2buf */
2131#define SET(x, y) do \
2132 { \
2133 if (remaining > 0) \
2134 { \
2135 x = NULL; /* initialize with null if tag doesn't exist */ \
2136 char* src = get_tag_string(entry, y); \
2137 if (src) \
2138 { \
2139 x = buf; \
2140 size_t len = strlcpy(buf, src, remaining) +1; \
2141 buf += len; remaining -= len; \
2142 } \
2143 } \
2144 } while(0)
2145
2146
2147 SET(id3->title, tag_title);
2148 SET(id3->artist, tag_artist);
2149 SET(id3->album, tag_album);
2150 SET(id3->genre_string, tag_genre);
2151 SET(id3->composer, tag_composer);
2152 SET(id3->comment, tag_comment);
2153 SET(id3->albumartist, tag_albumartist);
2154 SET(id3->grouping, tag_grouping);
2155
2156 id3->length = get_tag_numeric(entry, tag_length, idx_id);
2157 id3->playcount = get_tag_numeric(entry, tag_playcount, idx_id);
2158 id3->rating = get_tag_numeric(entry, tag_rating, idx_id);
2159 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
2160 id3->score = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
2161 id3->year = get_tag_numeric(entry, tag_year, idx_id);
2162
2163 id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
2164 id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
2165 id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
2166 if (id3->bitrate == 0)
2167 id3->bitrate = 1;
2168
2169 if (global_settings.autoresume_enable)
2170 {
2171 id3->elapsed = get_tag_numeric(entry, tag_lastelapsed, idx_id);
2172 logf("tagcache_fill_tags: Set elapsed for %s to %lX\n",
2173 id3->title, id3->elapsed);
2174
2175 id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id);
2176 logf("tagcache_fill_tags: Set offset for %s to %lX\n",
2177 id3->title, id3->offset);
2178 }
2179
2180 return true;
2181}
2182#endif /* defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) */
2183
2184static inline void write_item(const char *item)
2185{
2186 int len = strlen(item) + 1;
2187
2188 data_size += len;
2189 write(cachefd, item, len);
2190}
2191
2192static int check_if_empty(char **tag)
2193{
2194 int length;
2195
2196 if (*tag == NULL || **tag == '\0')
2197 {
2198 *tag = UNTAGGED;
2199 return sizeof(UNTAGGED); /* Tag length */
2200 }
2201
2202 length = strlen(*tag);
2203 if (length > TAG_MAXLEN)
2204 {
2205 logf("over length tag: %s", *tag);
2206 length = TAG_MAXLEN;
2207 str_setlen((*tag), length);
2208 }
2209
2210 return length + 1;
2211}
2212
2213#ifdef __PCTOOL__
2214static void db_log(const char *prefix, const char *msg)
2215{
2216 /* Crude logging for the sim - to aid in debugging */
2217 int logfd = open(ROCKBOX_DIR "/database.log",
2218 O_WRONLY | O_APPEND | O_CREAT, 0666);
2219 if (logfd >= 0)
2220 {
2221 write(logfd, prefix, strlen(prefix));
2222 write(logfd, ": ", 2);
2223 write(logfd, msg, strlen(msg));
2224 write(logfd, "\n", 1);
2225 close(logfd);
2226 }
2227}
2228#endif
2229
2230/* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
2231 * idea, as it uses lots of stack and is called from a recursive function
2232 * (check_dir).
2233 */
2234static void NO_INLINE add_tagcache(char *path, unsigned long mtime)
2235{
2236 #define ADD_TAG(entry, tag, data) \
2237 /* Adding tag */ \
2238 entry.tag_length[tag] = check_if_empty(data); \
2239 entry.tag_offset[tag] = offset; \
2240 offset += entry.tag_length[tag]
2241
2242 struct mp3entry id3;
2243 struct temp_file_entry entry;
2244 bool ret;
2245 int idx_id = -1;
2246 char tracknumfix[3];
2247 int offset = 0;
2248 int path_length = strlen(path);
2249 bool has_artist;
2250 bool has_grouping;
2251
2252 DB_LOG("file", path);
2253
2254 if (cachefd < 0)
2255 return ;
2256
2257 /* Check for overlength file path. */
2258 if (path_length > MAX_PATH || path_length > TAG_MAXLEN)
2259 {
2260 /* Path can't be shortened. */
2261 logf("Too long path: %s", path);
2262 DB_LOG("error", "path too long");
2263 return ;
2264 }
2265
2266 /* Check if the file is supported. */
2267 if (probe_file_format(path) == AFMT_UNKNOWN)
2268 return ;
2269
2270 /* Check if the file is already cached. */
2271#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
2272 idx_id = find_entry_ram(path);
2273#endif
2274
2275 /* Be sure the entry doesn't exist. */
2276 if (filenametag_fd >= 0 && idx_id < 0)
2277 idx_id = find_entry_disk(path, false);
2278
2279 /* Check if file has been modified. */
2280 if (idx_id >= 0)
2281 {
2282 struct index_entry idx;
2283
2284 /* TODO: Mark that the index exists (for fast reverse scan) */
2285 //found_idx[idx_id/8] |= idx_id%8;
2286
2287 if (!get_index(-1, idx_id, &idx, true))
2288 {
2289 logf("failed to retrieve index entry");
2290 DB_LOG("error", "failed to retrieve index entry");
2291 return ;
2292 }
2293
2294 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
2295 {
2296 /* No changes to file. */
2297 return ;
2298 }
2299
2300 /* Metadata might have been changed. Delete the entry. */
2301 logf("Re-adding: %s", path);
2302 DB_LOG("info", "re-adding");
2303 if (!delete_entry(idx_id))
2304 {
2305 logf("delete_entry failed: %d", idx_id);
2306 DB_LOG("error", "delete entry failed");
2307 return ;
2308 }
2309 }
2310
2311 /*memset(&id3, 0, sizeof(struct mp3entry)); -- get_metadata does this for us */
2312 memset(&entry, 0, sizeof(struct temp_file_entry));
2313 memset(&tracknumfix, 0, sizeof(tracknumfix));
2314 ret = get_metadata_ex(&id3, -1, path, METADATA_EXCLUDE_ID3_PATH);
2315
2316 if (!ret)
2317 {
2318 logf("get_metadata failed: %s", path);
2319 DB_LOG("error", "get_metadata failed");
2320 return ;
2321 }
2322
2323 logf("-> %s", path);
2324
2325 if (id3.tracknum <= 0) /* Track number missing? */
2326 {
2327 id3.tracknum = -1;
2328 }
2329
2330 /* Numeric tags */
2331 entry.tag_offset[tag_year] = id3.year;
2332 entry.tag_offset[tag_discnumber] = id3.discnum;
2333 entry.tag_offset[tag_tracknumber] = id3.tracknum;
2334 entry.tag_offset[tag_length] = id3.length;
2335 entry.tag_offset[tag_bitrate] = id3.bitrate;
2336 entry.tag_offset[tag_mtime] = mtime;
2337
2338 /* String tags. */
2339 has_artist = id3.artist != NULL
2340 && strlen(id3.artist) > 0;
2341 has_grouping = id3.grouping != NULL
2342 && strlen(id3.grouping) > 0;
2343
2344 ADD_TAG(entry, tag_filename, &path);
2345 ADD_TAG(entry, tag_title, &id3.title);
2346 ADD_TAG(entry, tag_artist, &id3.artist);
2347 ADD_TAG(entry, tag_album, &id3.album);
2348 ADD_TAG(entry, tag_genre, &id3.genre_string);
2349 ADD_TAG(entry, tag_composer, &id3.composer);
2350 ADD_TAG(entry, tag_comment, &id3.comment);
2351 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
2352 if (has_artist)
2353 {
2354 ADD_TAG(entry, tag_virt_canonicalartist, &id3.artist);
2355 }
2356 else
2357 {
2358 ADD_TAG(entry, tag_virt_canonicalartist, &id3.albumartist);
2359 }
2360 if (has_grouping)
2361 {
2362 ADD_TAG(entry, tag_grouping, &id3.grouping);
2363 }
2364 else
2365 {
2366 ADD_TAG(entry, tag_grouping, &id3.title);
2367 }
2368 entry.data_length = offset;
2369
2370 /* Write the header */
2371 write(cachefd, &entry, sizeof(struct temp_file_entry));
2372
2373 /* And tags also... Correct order is critical */
2374 write_item(path);
2375 write_item(id3.title);
2376 write_item(id3.artist);
2377 write_item(id3.album);
2378 write_item(id3.genre_string);
2379 write_item(id3.composer);
2380 write_item(id3.comment);
2381 write_item(id3.albumartist);
2382 if (has_artist)
2383 {
2384 write_item(id3.artist);
2385 }
2386 else
2387 {
2388 write_item(id3.albumartist);
2389 }
2390 if (has_grouping)
2391 {
2392 write_item(id3.grouping);
2393 }
2394 else
2395 {
2396 write_item(id3.title);
2397 }
2398
2399 total_entry_count++;
2400
2401 #undef ADD_TAG
2402}
2403#endif /*!defined(PLUGIN)*/
2404
2405
2406static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
2407{
2408 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2409 int len = strlen(str)+1;
2410 int i;
2411 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
2412 unsigned crc32 = 0xffffffff;
2413 char chr_lower;
2414 for (i = 0; str[i] != '\0' && i < len -1; i++)
2415 {
2416 chr_lower = tolower(str[i]);
2417 crc32 = crc_32(&chr_lower, 1, crc32);
2418 }
2419
2420 if (unique)
2421 {
2422 /* Check if the crc does not exist -> entry does not exist for sure. */
2423 for (i = 0; i < tempbufidx; i++)
2424 {
2425 if (crcbuf[-i] != crc32)
2426 continue;
2427
2428 if (!strcasecmp(str, index[i].str))
2429 {
2430 if (id < 0 || id >= lookup_buffer_depth)
2431 {
2432 logf("lookup buf overf.: %d", id);
2433 return false;
2434 }
2435
2436 lookup[id] = &index[i];
2437 return true;
2438 }
2439 }
2440 }
2441
2442 /* Insert to CRC buffer. */
2443 crcbuf[-tempbufidx] = crc32;
2444 tempbuf_left -= 4;
2445
2446 /* Insert it to the buffer. */
2447 tempbuf_left -= len;
2448 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count)
2449 {
2450 logf("temp buf error rem: %ld idx: %ld / %ld",
2451 tempbuf_left, tempbufidx, commit_entry_count-1);
2452 return false;
2453 }
2454 if (id >= lookup_buffer_depth)
2455 {
2456 logf("lookup buf overf. #2: %d", id);
2457 return false;
2458 }
2459
2460 if (id >= 0)
2461 {
2462 lookup[id] = &index[tempbufidx];
2463 index[tempbufidx].idlist.id = id;
2464 }
2465 else
2466 index[tempbufidx].idlist.id = -1;
2467
2468 index[tempbufidx].idlist.next = NULL;
2469 index[tempbufidx].idx_id = idx_id;
2470 index[tempbufidx].seek = -1;
2471 index[tempbufidx].str = &tempbuf[tempbuf_pos];
2472 memcpy(index[tempbufidx].str, str, len);
2473 tempbuf_pos += len;
2474 tempbufidx++;
2475
2476 return true;
2477}
2478
2479static int compare(const void *p1, const void *p2)
2480{
2481 do_timed_yield();
2482
2483 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2484 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2485
2486 if (strcmp(e1->str, UNTAGGED) == 0)
2487 {
2488 if (strcmp(e2->str, UNTAGGED) == 0)
2489 return 0;
2490 return -1;
2491 }
2492 else if (strcmp(e2->str, UNTAGGED) == 0)
2493 return 1;
2494
2495 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2496}
2497
2498static int tempbuf_sort(int fd)
2499{
2500 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2501 struct tagfile_entry fe;
2502 int i;
2503 int length;
2504
2505 /* Generate reverse lookup entries. */
2506 for (i = 0; i < lookup_buffer_depth; i++)
2507 {
2508 struct tempbuf_id_list *idlist;
2509
2510 if (!lookup[i])
2511 continue;
2512
2513 if (lookup[i]->idlist.id == i)
2514 continue;
2515
2516 idlist = &lookup[i]->idlist;
2517 while (idlist->next != NULL)
2518 idlist = idlist->next;
2519
2520 ALIGN_BUFFER(tempbuf_pos, tempbuf_left, alignof(struct tempbuf_id_list));
2521 tempbuf_left -= sizeof(struct tempbuf_id_list);
2522 if (tempbuf_left < 0)
2523 return -1;
2524
2525 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2526 tempbuf_pos += sizeof(struct tempbuf_id_list);
2527
2528 idlist = idlist->next;
2529 idlist->id = i;
2530 idlist->next = NULL;
2531
2532 do_timed_yield();
2533 }
2534
2535 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2536 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2537
2538 for (i = 0; i < tempbufidx; i++)
2539 {
2540 struct tempbuf_id_list *idlist = &index[i].idlist;
2541
2542 /* Fix the lookup list. */
2543 while (idlist != NULL)
2544 {
2545 if (idlist->id >= 0)
2546 lookup[idlist->id] = &index[i];
2547 idlist = idlist->next;
2548 }
2549
2550 index[i].seek = lseek(fd, 0, SEEK_CUR);
2551 length = strlen(index[i].str) + 1;
2552 fe.tag_length = length;
2553 fe.idx_id = index[i].idx_id;
2554
2555 /* Check the chunk alignment. */
2556 if ((fe.tag_length + sizeof(struct tagfile_entry))
2557 % TAGFILE_ENTRY_CHUNK_LENGTH)
2558 {
2559 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2560 ((fe.tag_length + sizeof(struct tagfile_entry))
2561 % TAGFILE_ENTRY_CHUNK_LENGTH);
2562 }
2563
2564 if (write_tagfile_entry(fd, &fe) != sizeof(struct tagfile_entry))
2565 {
2566 logf("tempbuf_sort: write error #1");
2567 return -1;
2568 }
2569
2570 if (write(fd, index[i].str, length) != length)
2571 {
2572 logf("tempbuf_sort: write error #2");
2573 return -2;
2574 }
2575
2576 /* Write some padding. */
2577 if (fe.tag_length - length > 0)
2578 write(fd, "XXXXXXXX", fe.tag_length - length);
2579 }
2580
2581 return i;
2582}
2583
2584inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2585{
2586 if (id < 0 || id >= lookup_buffer_depth)
2587 return NULL;
2588
2589 return lookup[id];
2590}
2591
2592
2593inline static int tempbuf_find_location(int id)
2594{
2595 struct tempbuf_searchidx *entry;
2596
2597 entry = tempbuf_locate(id);
2598 if (entry == NULL)
2599 return -1;
2600
2601 return entry->seek;
2602}
2603
2604static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2605{
2606 struct master_header tcmh;
2607 struct index_entry idx;
2608 int masterfd;
2609 int masterfd_pos;
2610 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2611 int max_entries;
2612 int entries_processed = 0;
2613 int i, j;
2614
2615 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2616
2617 logf("Building numeric indices...");
2618 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2619
2620 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2621 return false;
2622
2623 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2624 SEEK_CUR);
2625 if (masterfd_pos < 0)
2626 {
2627 logf("we can't append!");
2628 close(masterfd);
2629 return false;
2630 }
2631
2632 while (entries_processed < h->entry_count && !USR_CANCEL)
2633 {
2634 int count = MIN(h->entry_count - entries_processed, max_entries);
2635
2636 /* Read in as many entries as possible. */
2637 for (i = 0; i < count; i++)
2638 {
2639 struct temp_file_entry *tfe = &entrybuf[i];
2640 int datastart;
2641
2642 /* Read in numeric data. */
2643 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2644 sizeof(struct temp_file_entry))
2645 {
2646 logf("read fail #1");
2647 close(masterfd);
2648 return false;
2649 }
2650
2651 datastart = lseek(tmpfd, 0, SEEK_CUR);
2652
2653 /**
2654 * Read string data from the following tags:
2655 * - tag_filename
2656 * - tag_artist
2657 * - tag_album
2658 * - tag_title
2659 *
2660 * A crc32 hash is calculated from the read data
2661 * and stored back to the data offset field kept in memory.
2662 */
2663#define tmpdb_read_string_tag(tag) \
2664 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2665 if ((unsigned long)tfe->tag_length[tag] > (unsigned long)build_idx_bufsz) \
2666 { \
2667 logf("read fail: buffer overflow"); \
2668 close(masterfd); \
2669 return false; \
2670 } \
2671 \
2672 if (read(tmpfd, build_idx_buf, tfe->tag_length[tag]) != \
2673 tfe->tag_length[tag]) \
2674 { \
2675 logf("read fail #2"); \
2676 close(masterfd); \
2677 return false; \
2678 } \
2679 str_setlen(build_idx_buf, tfe->tag_length[tag]); \
2680 \
2681 tfe->tag_offset[tag] = crc_32(build_idx_buf, strlen(build_idx_buf), 0xffffffff); \
2682 lseek(tmpfd, datastart, SEEK_SET)
2683
2684 tmpdb_read_string_tag(tag_filename);
2685 tmpdb_read_string_tag(tag_artist);
2686 tmpdb_read_string_tag(tag_album);
2687 tmpdb_read_string_tag(tag_title);
2688
2689 /* Seek to the end of the string data. */
2690 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2691 }
2692
2693 /* Backup the master index position. */
2694 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2695 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2696
2697 /* Check if we can resurrect some deleted runtime statistics data. */
2698 for (i = 0; i < tcmh.tch.entry_count && !USR_CANCEL; i++)
2699 {
2700 /* Read the index entry. */
2701 if (read_index_entries(masterfd, &idx, 1) != sizeof(struct index_entry))
2702 {
2703 logf("read fail #3");
2704 close(masterfd);
2705 return false;
2706 }
2707
2708 /**
2709 * Skip unless the entry is marked as being deleted
2710 * or the data has already been resurrected.
2711 */
2712 if (!(idx.flag & FLAG_DELETED) || (idx.flag & FLAG_RESURRECTED))
2713 continue;
2714
2715 /* Now try to match the entry. */
2716 /**
2717 * To succesfully match a song, the following conditions
2718 * must apply:
2719 *
2720 * For numeric fields: tag_length
2721 * - Full identical match is required
2722 *
2723 * If tag_filename matches, no further checking necessary.
2724 *
2725 * For string hashes: tag_artist, tag_album, tag_title
2726 * - All three of these must match
2727 */
2728 for (j = 0; j < count; j++)
2729 {
2730 struct temp_file_entry *tfe = &entrybuf[j];
2731
2732 /* Try to match numeric fields first. */
2733 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2734 continue;
2735
2736 /* Now it's time to do the hash matching. */
2737 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2738 {
2739 int match_count = 0;
2740
2741 /* No filename match, check if we can match two other tags. */
2742#define tmpdb_match(tag) \
2743 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2744 match_count++
2745
2746 tmpdb_match(tag_artist);
2747 tmpdb_match(tag_album);
2748 tmpdb_match(tag_title);
2749
2750 if (match_count < 3)
2751 {
2752 /* Still no match found, give up. */
2753 continue;
2754 }
2755 }
2756
2757 /* A match found, now copy & resurrect the statistical data. */
2758#define tmpdb_copy_tag(tag) \
2759 tfe->tag_offset[tag] = idx.tag_seek[tag]
2760
2761 tmpdb_copy_tag(tag_playcount);
2762 tmpdb_copy_tag(tag_rating);
2763 tmpdb_copy_tag(tag_playtime);
2764 tmpdb_copy_tag(tag_lastplayed);
2765 tmpdb_copy_tag(tag_commitid);
2766 tmpdb_copy_tag(tag_lastelapsed);
2767 tmpdb_copy_tag(tag_lastoffset);
2768
2769 /* Avoid processing this entry again. */
2770 idx.flag |= FLAG_RESURRECTED;
2771
2772 lseek(masterfd, -(off_t)sizeof(struct index_entry), SEEK_CUR);
2773 if (write_index_entries(masterfd, &idx, 1) != sizeof(struct index_entry))
2774 {
2775 logf("masterfd writeback fail #1");
2776 close(masterfd);
2777 return false;
2778 }
2779
2780 logf("Entry resurrected");
2781 }
2782 }
2783
2784
2785 /* Restore the master index position. */
2786 lseek(masterfd, masterfd_pos, SEEK_SET);
2787
2788 /* Commit the data to the index. */
2789 for (i = 0; i < count && !USR_CANCEL; i++)
2790 {
2791 int loc = lseek(masterfd, 0, SEEK_CUR);
2792
2793 if (read_index_entries(masterfd, &idx, 1) != sizeof(struct index_entry))
2794 {
2795 logf("read fail #3");
2796 close(masterfd);
2797 return false;
2798 }
2799
2800 for (j = 0; j < TAG_COUNT; j++)
2801 {
2802 if (!TAGCACHE_IS_NUMERIC(j))
2803 continue;
2804
2805 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2806 }
2807 idx.flag = entrybuf[i].flag;
2808
2809 if (idx.tag_seek[tag_commitid])
2810 {
2811 /* Data has been resurrected. */
2812 idx.flag |= FLAG_DIRTYNUM;
2813 }
2814 else if (tc_stat.ready && current_tcmh.commitid > 0)
2815 {
2816 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2817 idx.flag |= FLAG_DIRTYNUM;
2818 }
2819
2820 /* Write back the updated index. */
2821 lseek(masterfd, loc, SEEK_SET);
2822 if (write_index_entries(masterfd, &idx, 1) != sizeof(struct index_entry))
2823 {
2824 logf("write fail");
2825 close(masterfd);
2826 return false;
2827 }
2828 }
2829
2830 entries_processed += count;
2831 logf("%d/%" PRId32 " entries processed", entries_processed, h->entry_count);
2832 }
2833
2834 close(masterfd);
2835
2836 return true;
2837}
2838
2839/**
2840 * Return values:
2841 * > 0 success
2842 * == 0 temporary failure
2843 * < 0 fatal error
2844 */
2845static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2846{
2847 int i;
2848 struct tagcache_header tch;
2849 struct master_header tcmh;
2850 struct index_entry idxbuf[IDX_BUF_DEPTH];
2851 int idxbuf_pos;
2852 int fd = -1, masterfd;
2853 bool error = false;
2854 int init;
2855 int masterfd_pos;
2856
2857 logf("Building index: %d", index_type);
2858
2859 /* Check the number of entries we need to allocate ram for. */
2860 commit_entry_count = h->entry_count + 1;
2861
2862 masterfd = open_master_fd(&tcmh, false);
2863 if (masterfd >= 0)
2864 {
2865 commit_entry_count += tcmh.tch.entry_count;
2866 close(masterfd);
2867 }
2868 else
2869 remove_files(); /* Just to be sure we are clean. */
2870
2871 /* Open the index file, which contains the tag names. */
2872 fd = open_tag_fd(&tch, index_type, true);
2873 if (fd >= 0)
2874 {
2875 logf("tch.datasize=%" PRId32, tch.datasize);
2876 lookup_buffer_depth = 1 +
2877 /* First part */ commit_entry_count +
2878 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2879 }
2880 else
2881 {
2882 lookup_buffer_depth = 1 +
2883 /* First part */ commit_entry_count +
2884 /* Second part */ 0;
2885 }
2886
2887 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2888 logf("commit_entry_count=%ld", commit_entry_count);
2889
2890 /* Allocate buffer for all index entries from both old and new
2891 * tag files. */
2892 tempbufidx = 0;
2893 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2894
2895 /* Allocate lookup buffer. The first portion of commit_entry_count
2896 * contains the new tags in the temporary file and the second
2897 * part for locating entries already in the db.
2898 *
2899 * New tags Old tags
2900 * +---------+---------------------------+
2901 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2902 * +---------+---------------------------+
2903 *
2904 * Old tags are inserted to a temporary buffer with position:
2905 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2906 * And new tags with index:
2907 * tempbuf_insert(idx, ...);
2908 *
2909 * The buffer is sorted and written into tag file:
2910 * tempbuf_sort(...);
2911 * leaving master index locations messed up.
2912 *
2913 * That is fixed using the lookup buffer for old tags:
2914 * new_seek = tempbuf_find_location(old_seek, ...);
2915 * and for new tags:
2916 * new_seek = tempbuf_find_location(idx);
2917 */
2918 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2919 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2920 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2921
2922 /* And calculate the remaining data space used mainly for storing
2923 * tag data (strings). */
2924 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2925 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2926 {
2927 logf("Buffer way too small!");
2928 close(fd);
2929 return 0;
2930 }
2931
2932 if (fd >= 0)
2933 {
2934 /**
2935 * If tag file contains unique tags (sorted index), we will load
2936 * it entirely into memory so we can resort it later for use with
2937 * chunked browsing.
2938 */
2939 if (TAGCACHE_IS_SORTED(index_type))
2940 {
2941 logf("loading tags...");
2942 for (i = 0; i < tch.entry_count && !USR_CANCEL; i++)
2943 {
2944 struct tagfile_entry entry;
2945 int loc = lseek(fd, 0, SEEK_CUR);
2946 bool ret;
2947 switch (read_tagfile_entry_and_tag(fd, &entry, build_idx_buf, build_idx_bufsz))
2948 {
2949 case e_SUCCESS_LEN_ZERO: /* Skip deleted entries. */
2950 continue;
2951 case e_SUCCESS:
2952 break;
2953 case e_ENTRY_SIZEMISMATCH:
2954 logf("read error #7");
2955 close(fd);
2956 return -2;
2957 case e_TAG_TOOLONG:
2958 logf("too long tag #3");
2959 close(fd);
2960 return -2;
2961 case e_TAG_SIZEMISMATCH:
2962 logf("read error #8");
2963 close(fd);
2964 return -2;
2965 }
2966
2967 /**
2968 * Save the tag and tag id in the memory buffer. Tag id
2969 * is saved so we can later reindex the master lookup
2970 * table when the index gets resorted.
2971 */
2972 ret = tempbuf_insert(build_idx_buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2973 + commit_entry_count, entry.idx_id,
2974 TAGCACHE_IS_UNIQUE(index_type));
2975 if (!ret)
2976 {
2977 close(fd);
2978 return -3;
2979 }
2980 do_timed_yield();
2981 }
2982 logf("done");
2983 }
2984 else
2985 tempbufidx = tch.entry_count;
2986 }
2987 else
2988 {
2989 logf("Create New Index: %d", index_type);
2990 /**
2991 * Creating new index file to store the tags. No need to preload
2992 * anything whether the index type is sorted or not.
2993 *
2994 * Note: although we are creating a file under the db path, it must
2995 * already exist by this point so no mkdir is required.
2996 */
2997 fd = open_pathfmt(build_idx_buf, build_idx_bufsz,
2998 O_WRONLY | O_CREAT | O_TRUNC,
2999 "%s/" TAGCACHE_FILE_INDEX,
3000 tc_stat.db_path, index_type);
3001 if (fd < 0)
3002 {
3003 logf(TAGCACHE_FILE_INDEX " open fail", index_type);
3004 return -2;
3005 }
3006
3007 tch.magic = TAGCACHE_MAGIC;
3008 tch.entry_count = 0;
3009 tch.datasize = 0;
3010
3011 if (write_tagcache_header(fd, &tch) != sizeof(struct tagcache_header))
3012 {
3013 logf("header write failed");
3014 close(fd);
3015 return -2;
3016 }
3017 }
3018
3019 /* Loading the tag lookup file as "master file". */
3020 logf("Loading index file");
3021 masterfd = open_db_fd(TAGCACHE_FILE_MASTER, O_RDWR);
3022
3023 if (masterfd < 0)
3024 {
3025 logf("Creating new DB");
3026 masterfd = open_db_fd(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
3027
3028 if (masterfd < 0)
3029 {
3030 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
3031 close(fd);
3032 return -2;
3033 }
3034
3035 /* Write the header (write real values later). */
3036 memset(&tcmh, 0, sizeof(struct master_header));
3037 tcmh.tch = *h;
3038 tcmh.tch.entry_count = 0;
3039 tcmh.tch.datasize = 0;
3040 tcmh.dirty = true;
3041 write_master_header(masterfd, &tcmh);
3042 init = true;
3043 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
3044 }
3045 else
3046 {
3047 /**
3048 * Master file already exists so we need to process the current
3049 * file first.
3050 */
3051 init = false;
3052
3053 if (read_master_header(masterfd, &tcmh) != sizeof(struct master_header) ||
3054 tcmh.tch.magic != TAGCACHE_MAGIC)
3055 {
3056 logf("header error");
3057 close(fd);
3058 close(masterfd);
3059 return -2;
3060 }
3061
3062 /**
3063 * If we reach end of the master file, we need to expand it to
3064 * hold new tags. If the current index is not sorted, we can
3065 * simply append new data to end of the file.
3066 * However, if the index is sorted, we need to update all tag
3067 * pointers in the master file for the current index.
3068 */
3069 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
3070 SEEK_CUR);
3071 if (masterfd_pos == filesize(masterfd))
3072 {
3073 logf("appending...");
3074 init = true;
3075 }
3076 }
3077
3078 /**
3079 * Load new unique tags in memory to be sorted later and added
3080 * to the master lookup file.
3081 */
3082 if (TAGCACHE_IS_SORTED(index_type))
3083 {
3084 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
3085 /* h is the header of the temporary file containing new tags. */
3086 logf("inserting new tags...");
3087 for (i = 0; i < h->entry_count && !USR_CANCEL; i++)
3088 {
3089 struct temp_file_entry entry;
3090
3091 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
3092 sizeof(struct temp_file_entry))
3093 {
3094 logf("read fail #3");
3095 error = true;
3096 goto error_exit;
3097 }
3098
3099 /* Read data. */
3100 if (entry.tag_length[index_type] >= build_idx_bufsz)
3101 {
3102 logf("too long entry!");
3103 error = true;
3104 goto error_exit;
3105 }
3106
3107 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
3108 if (read(tmpfd, build_idx_buf, entry.tag_length[index_type]) !=
3109 entry.tag_length[index_type])
3110 {
3111 logf("read fail #4");
3112 error = true;
3113 goto error_exit;
3114 }
3115 str_setlen(build_idx_buf, entry.tag_length[index_type]);
3116
3117#if defined(PLUGIN)
3118 if (user_check_tag(index_type, build_idx_buf))
3119#endif /*defined(PLUGIN)*/
3120 {
3121 if (TAGCACHE_IS_UNIQUE(index_type))
3122 error = !tempbuf_insert(build_idx_buf, i, -1, true);
3123 else
3124 error = !tempbuf_insert(build_idx_buf, i,
3125 tcmh.tch.entry_count + i, false);
3126
3127 if (error)
3128 {
3129 logf("insert error");
3130 goto error_exit;
3131 }
3132 }
3133 /* Skip to next. */
3134 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
3135 entry.tag_length[index_type], SEEK_CUR);
3136 do_timed_yield();
3137 }
3138 logf("done");
3139
3140 /* Sort the buffer data and write it to the index file. */
3141 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
3142 /**
3143 * We need to truncate the index file now. There can be junk left
3144 * at the end of file (however, we _should_ always follow the
3145 * entry_count and don't crash with that).
3146 */
3147 ftruncate(fd, lseek(fd, 0, SEEK_CUR));
3148
3149 i = tempbuf_sort(fd);
3150 if (i < 0)
3151 goto error_exit;
3152 logf("sorted %d tags", i);
3153
3154 /**
3155 * Now update all indexes in the master lookup file.
3156 */
3157 logf("updating indices...");
3158 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3159 for (i = 0; i < tcmh.tch.entry_count && !USR_CANCEL; i += idxbuf_pos)
3160 {
3161 int j;
3162 int loc = lseek(masterfd, 0, SEEK_CUR);
3163
3164 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
3165
3166 if (read_index_entries(masterfd, idxbuf, idxbuf_pos) !=
3167 (ssize_t)sizeof(struct index_entry) * idxbuf_pos)
3168 {
3169 logf("read fail #5");
3170 error = true;
3171 goto error_exit ;
3172 }
3173 lseek(masterfd, loc, SEEK_SET);
3174
3175 for (j = 0; j < idxbuf_pos; j++)
3176 {
3177 if (idxbuf[j].flag & FLAG_DELETED)
3178 {
3179 /* We can just ignore deleted entries. */
3180 // idxbuf[j].tag_seek[index_type] = 0;
3181 continue;
3182 }
3183
3184 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
3185 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
3186 + commit_entry_count);
3187
3188 if (idxbuf[j].tag_seek[index_type] < 0)
3189 {
3190 logf("update error: %" PRId32 "/%d/%" PRId32,
3191 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
3192 error = true;
3193 goto error_exit;
3194 }
3195
3196 do_timed_yield();
3197 }
3198
3199 /* Write back the updated index. */
3200 if (write_index_entries(masterfd, idxbuf, idxbuf_pos) !=
3201 (ssize_t)sizeof(struct index_entry) * idxbuf_pos)
3202 {
3203 logf("write fail");
3204 error = true;
3205 goto error_exit;
3206 }
3207 }
3208 logf("done");
3209 }
3210
3211 /**
3212 * Walk through the temporary file containing the new tags.
3213 */
3214 // build_normal_index(h, tmpfd, masterfd, idx);
3215 logf("updating new indices...");
3216 lseek(masterfd, masterfd_pos, SEEK_SET);
3217 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
3218 lseek(fd, 0, SEEK_END);
3219 for (i = 0; i < h->entry_count && !USR_CANCEL; i += idxbuf_pos)
3220 {
3221 int j;
3222
3223 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
3224 if (init)
3225 {
3226 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
3227 }
3228 else
3229 {
3230 int loc = lseek(masterfd, 0, SEEK_CUR);
3231
3232 if (read_index_entries(masterfd, idxbuf, idxbuf_pos) !=
3233 (ssize_t)sizeof(struct index_entry) * idxbuf_pos)
3234 {
3235 logf("read fail #6");
3236 error = true;
3237 break ;
3238 }
3239 lseek(masterfd, loc, SEEK_SET);
3240 }
3241
3242 /* Read entry headers. */
3243 for (j = 0; j < idxbuf_pos; j++)
3244 {
3245 if (!TAGCACHE_IS_SORTED(index_type))
3246 {
3247 struct temp_file_entry entry;
3248 struct tagfile_entry fe;
3249
3250 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
3251 sizeof(struct temp_file_entry))
3252 {
3253 logf("read fail #7");
3254 error = true;
3255 break ;
3256 }
3257
3258 /* Read data. */
3259 if (entry.tag_length[index_type] >= build_idx_bufsz)
3260 {
3261 logf("too long entry!");
3262 logf("length=%d", entry.tag_length[index_type]);
3263 logf("pos=0x%02lx", (unsigned long) lseek(tmpfd, 0, SEEK_CUR));
3264 error = true;
3265 break ;
3266 }
3267
3268 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
3269 if (read(tmpfd, build_idx_buf, entry.tag_length[index_type]) !=
3270 entry.tag_length[index_type])
3271 {
3272 logf("read fail #8");
3273 logf("offset=0x%02" PRIx32, entry.tag_offset[index_type]);
3274 logf("length=0x%02x", entry.tag_length[index_type]);
3275 error = true;
3276 break ;
3277 }
3278
3279 /* Write to index file. */
3280 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
3281 fe.tag_length = entry.tag_length[index_type];
3282 fe.idx_id = tcmh.tch.entry_count + i + j;
3283 write_tagfile_entry(fd, &fe);
3284 write(fd, build_idx_buf, fe.tag_length);
3285 tempbufidx++;
3286
3287 /* Skip to next. */
3288 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
3289 entry.tag_length[index_type], SEEK_CUR);
3290 }
3291 else
3292 {
3293 /* Locate the correct entry from the sorted array. */
3294 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
3295 if (idxbuf[j].tag_seek[index_type] < 0)
3296 {
3297 logf("entry not found (%d)", j);
3298 error = true;
3299 break ;
3300 }
3301 }
3302 }
3303
3304 /* Write index. */
3305 if (write_index_entries(masterfd, idxbuf, idxbuf_pos) !=
3306 (ssize_t)sizeof(struct index_entry) * idxbuf_pos)
3307 {
3308 logf("tagcache: write fail #4");
3309 error = true;
3310 break ;
3311 }
3312
3313 do_timed_yield();
3314 }
3315 logf("done");
3316
3317 /* Finally write the header. */
3318 tch.magic = TAGCACHE_MAGIC;
3319 tch.entry_count = tempbufidx;
3320 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
3321 lseek(fd, 0, SEEK_SET);
3322 write_tagcache_header(fd, &tch);
3323
3324 if (index_type != tag_filename)
3325 h->datasize += tch.datasize;
3326 logf("s:%d/%" PRId32 "/%" PRId32, index_type, tch.datasize, h->datasize);
3327 error_exit:
3328
3329 close(fd);
3330 close(masterfd);
3331
3332 if (error)
3333 return -2;
3334
3335 return 1;
3336}
3337
3338static bool commit(void)
3339{
3340 struct tagcache_header tch;
3341 struct master_header tcmh;
3342 int i, len, rc;
3343 int tmpfd;
3344 int masterfd;
3345#ifdef HAVE_DIRCACHE
3346 bool dircache_buffer_stolen = false;
3347#endif
3348#ifdef HAVE_TC_RAMCACHE
3349 bool ramcache_buffer_stolen = false;
3350#endif
3351 logf("committing tagcache");
3352
3353 while (write_lock)
3354 sleep(1);
3355
3356#if !defined(PLUGIN)
3357 int fd = open_db_fd(TAGCACHE_FILE_NOCOMMIT, O_RDONLY);
3358 if (fd >= 0)
3359 {
3360 logf("canceling commit");
3361 tc_stat.commit_delayed = true;
3362 close(fd);
3363 tmpfd = -1;
3364 }
3365 else
3366#endif /*!defined(PLUGIN)*/
3367 {
3368 tmpfd = open_db_fd(TAGCACHE_FILE_TEMP, O_RDONLY);
3369 }
3370 if (tmpfd < 0)
3371 {
3372 logf("nothing to commit");
3373 return true;
3374 }
3375
3376
3377 /* Load the header. */
3378 len = sizeof(struct tagcache_header);
3379 rc = read(tmpfd, &tch, len);
3380
3381 if (tch.magic != TAGCACHE_MAGIC || rc != len)
3382 {
3383 logf("incorrect tmpheader");
3384 close(tmpfd);
3385 remove_db_file(TAGCACHE_FILE_TEMP);
3386 return false;
3387 }
3388
3389 if (tch.entry_count == 0)
3390 logf("nothing to commit");
3391
3392 /* Fully initialize existing headers (if any) before going further. */
3393 tc_stat.ready = check_all_headers();
3394
3395#ifdef HAVE_EEPROM_SETTINGS
3396 remove_db_file(TAGCACHE_STATEFILE);
3397#endif
3398
3399 /* At first be sure to unload the ramcache! */
3400#ifdef HAVE_TC_RAMCACHE
3401 tc_stat.ramcache = false;
3402#endif
3403
3404 /* Beyond here, jump to commit_error to undo locks and restore dircache */
3405 rc = false;
3406 read_lock++;
3407
3408 /* Try to steal every buffer we can :) */
3409#ifdef HAVE_DIRCACHE
3410 if (tempbuf_size == 0)
3411 {
3412 /* Suspend dircache to free its allocation. */
3413 dircache_free_buffer();
3414 dircache_buffer_stolen = true;
3415
3416 allocate_tempbuf();
3417 }
3418#endif /* HAVE_DIRCACHE */
3419
3420#ifdef HAVE_TC_RAMCACHE
3421 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
3422 {
3423 tcrc_buffer_lock();
3424 tempbuf = (char *)(tcramcache.hdr + 1);
3425 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
3426 tempbuf_size &= ~0x03;
3427 ramcache_buffer_stolen = true;
3428 }
3429#endif /* HAVE_TC_RAMCACHE */
3430
3431#if defined(PLUGIN)
3432 if (tempbuf_size == 0)
3433 {
3434 tempbuf = rb->plugin_get_audio_buffer(&tempbuf_size);
3435 tempbuf_size &= ~0x03;
3436 }
3437#endif /*defined(PLUGIN)*/
3438
3439 /* And finally fail if there are no buffers available. */
3440 if (tempbuf_size == 0)
3441 {
3442 logf("delaying commit until next boot");
3443 tc_stat.commit_delayed = true;
3444 close(tmpfd);
3445 goto commit_error;
3446 }
3447
3448 logf("commit %" PRId32 " entries...", tch.entry_count);
3449
3450 /* Mark DB dirty so it will stay disabled if commit fails. */
3451 current_tcmh.dirty = true;
3452 update_master_header();
3453
3454 /* Now create the index files. */
3455 tc_stat.commit_step = 0;
3456 tch.datasize = 0;
3457 tc_stat.commit_delayed = false;
3458
3459 for (i = 0; i < TAG_COUNT && !USR_CANCEL; i++)
3460 {
3461 int ret;
3462
3463 if (TAGCACHE_IS_NUMERIC(i))
3464 continue;
3465
3466 tc_stat.commit_step++;
3467 ret = build_index(i, &tch, tmpfd);
3468 if (ret <= 0)
3469 {
3470 close(tmpfd);
3471 logf("tagcache failed init");
3472 if (ret == 0)
3473 tc_stat.commit_delayed = true;
3474
3475 tc_stat.commit_step = 0;
3476 goto commit_error;
3477 }
3478 do_timed_yield();
3479 }
3480
3481 if (!build_numeric_indices(&tch, tmpfd))
3482 {
3483 logf("Failure to commit numeric indices");
3484 close(tmpfd);
3485 tc_stat.commit_step = 0;
3486 goto commit_error;
3487 }
3488
3489 close(tmpfd);
3490
3491 tc_stat.commit_step = 0;
3492
3493 if (!USR_CANCEL)
3494 {
3495 /* Update the master index headers. */
3496 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3497 goto commit_error;
3498
3499 remove_db_file(TAGCACHE_FILE_TEMP);
3500
3501 tcmh.tch.entry_count += tch.entry_count;
3502 tcmh.tch.datasize = sizeof(struct master_header)
3503 + sizeof(struct index_entry) * tcmh.tch.entry_count
3504 + tch.datasize;
3505 tcmh.dirty = false;
3506 tcmh.commitid++;
3507
3508 lseek(masterfd, 0, SEEK_SET);
3509 write_master_header(masterfd, &tcmh);
3510 close(masterfd);
3511
3512 logf("tagcache committed");
3513 tagcache_commit_finalize();
3514
3515#if defined(HAVE_TC_RAMCACHE)
3516 if (ramcache_buffer_stolen)
3517 {
3518 tempbuf = NULL;
3519 tempbuf_size = 0;
3520 ramcache_buffer_stolen = false;
3521 tcrc_buffer_unlock();
3522 }
3523
3524 /* Reload tagcache. */
3525 if (tc_stat.ramcache_allocated > 0)
3526 tagcache_start_scan();
3527#endif /* HAVE_TC_RAMCACHE */
3528
3529 rc = true;
3530 } /*!USR_CANCEL*/
3531
3532commit_error:
3533#ifdef HAVE_TC_RAMCACHE
3534 if (ramcache_buffer_stolen)
3535 {
3536 tempbuf = NULL;
3537 tempbuf_size = 0;
3538 tcrc_buffer_unlock();
3539 }
3540#endif /* HAVE_TC_RAMCACHE */
3541
3542 read_lock--;
3543
3544#ifdef HAVE_DIRCACHE
3545 /* Resume the dircache, if we stole the buffer. */
3546 if (dircache_buffer_stolen)
3547 {
3548 free_tempbuf();
3549 dircache_resume();
3550 }
3551#endif /* HAVE_DIRCACHE */
3552
3553 return rc;
3554}
3555
3556void tagcache_commit_finalize(void)
3557{
3558 tc_stat.ready = check_all_headers();
3559 tc_stat.readyvalid = true;
3560}
3561
3562#if !defined(PLUGIN)
3563#ifndef __PCTOOL__
3564
3565static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3566{
3567 struct index_entry idx;
3568
3569 if (!tc_stat.ready)
3570 return false;
3571
3572 if (!TAGCACHE_IS_NUMERIC(tag))
3573 return false;
3574
3575 if (!get_index(masterfd, idx_id, &idx, false))
3576 return false;
3577
3578 idx.tag_seek[tag] = data;
3579 idx.flag |= FLAG_DIRTYNUM;
3580
3581 return write_index(masterfd, idx_id, &idx);
3582}
3583
3584#if 0
3585bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3586 int tag, long data)
3587{
3588 struct master_header myhdr;
3589
3590 if (tcs->masterfd < 0)
3591 {
3592 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3593 return false;
3594 }
3595
3596 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3597}
3598#endif
3599
3600static bool command_queue_is_full(void)
3601{
3602 int next;
3603
3604 next = command_queue_widx + 1;
3605 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3606 next = 0;
3607
3608 return (next == command_queue_ridx);
3609}
3610
3611static void command_queue_sync_callback(void)
3612{
3613 struct master_header myhdr;
3614 int masterfd;
3615
3616 mutex_lock(&command_queue_mutex);
3617
3618 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3619 return;
3620
3621 while (command_queue_ridx != command_queue_widx)
3622 {
3623 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3624
3625 switch (ce->command)
3626 {
3627 case CMD_UPDATE_MASTER_HEADER:
3628 {
3629 close(masterfd);
3630 update_master_header();
3631
3632 /* Re-open the masterfd. */
3633 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3634 return;
3635
3636 break;
3637 }
3638 case CMD_UPDATE_NUMERIC:
3639 {
3640 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3641 break;
3642 }
3643 }
3644
3645 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3646 command_queue_ridx = 0;
3647 }
3648
3649 close(masterfd);
3650
3651 tc_stat.queue_length = 0;
3652 mutex_unlock(&command_queue_mutex);
3653}
3654
3655static void run_command_queue(bool force)
3656{
3657 if (COMMAND_QUEUE_IS_EMPTY)
3658 return;
3659
3660 if (force || command_queue_is_full())
3661 command_queue_sync_callback();
3662 else
3663 register_storage_idle_func(command_queue_sync_callback);
3664}
3665
3666static void queue_command(int cmd, long idx_id, int tag, long data)
3667{
3668 while (1)
3669 {
3670 int next;
3671
3672 mutex_lock(&command_queue_mutex);
3673 next = command_queue_widx + 1;
3674 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3675 next = 0;
3676
3677 /* Make sure queue is not full. */
3678 if (next != command_queue_ridx)
3679 {
3680 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3681
3682 ce->command = cmd;
3683 ce->idx_id = idx_id;
3684 ce->tag = tag;
3685 ce->data = data;
3686
3687 command_queue_widx = next;
3688
3689 tc_stat.queue_length++;
3690
3691 mutex_unlock(&command_queue_mutex);
3692 break;
3693 }
3694
3695 /* Queue is full, try again later... */
3696 mutex_unlock(&command_queue_mutex);
3697 sleep(1);
3698 }
3699}
3700
3701long tagcache_increase_serial(void)
3702{
3703 long old;
3704
3705 if (!tc_stat.ready)
3706 return -2;
3707
3708 while (read_lock)
3709 sleep(1);
3710
3711 old = current_tcmh.serial++;
3712 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3713
3714 return old;
3715}
3716
3717void tagcache_update_numeric(int idx_id, int tag, long data)
3718{
3719 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3720}
3721#endif /* !__PCTOOL__ */
3722
3723static bool write_tag(int fd, const char *tagstr, const char *datastr)
3724{
3725 char buf[512];
3726 const int bufsz = sizeof(buf);
3727 int i;
3728
3729 snprintf(buf, bufsz, "%s=\"", tagstr);
3730
3731 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3732 {
3733 if (*datastr == '\0')
3734 break;
3735
3736 if (*datastr == '"' || *datastr == '\\')
3737 buf[i++] = '\\';
3738
3739 else if (*datastr == '\n')
3740 {
3741 buf[i++] = '\\';
3742 buf[i] = 'n';
3743 continue;
3744 }
3745
3746 buf[i] = *(datastr++);
3747 }
3748
3749 str_setlen(buf, bufsz - 1);
3750 strmemccpy(&buf[i], "\" ", (bufsz - i - 1));
3751
3752 write(fd, buf, i + 2);
3753
3754 return true;
3755}
3756
3757#ifndef __PCTOOL__
3758
3759static bool read_tag(char *dest, long size,
3760 const char *src, const char *tagstr)
3761{
3762 int pos;
3763 char current_tag[32];
3764
3765 while (*src != '\0')
3766 {
3767 /* Skip all whitespace */
3768 while (*src == ' ')
3769 src++;
3770
3771 if (*src == '\0')
3772 break;
3773
3774 pos = 0;
3775 /* Read in tag name */
3776 while (*src != '=' && *src != ' ')
3777 {
3778 current_tag[pos] = *src;
3779 src++;
3780 pos++;
3781
3782 if (*src == '\0' || pos >= (int) sizeof(current_tag))
3783 return false;
3784 }
3785
3786 str_setlen(current_tag, pos);
3787
3788 /* Read in tag data */
3789
3790 /* Find the start. */
3791 while (*src != '"' && *src != '\0')
3792 src++;
3793
3794 if (*src == '\0' || *(++src) == '\0')
3795 return false;
3796
3797 /* Read the data. */
3798 for (pos = 0; pos < size; pos++)
3799 {
3800 if (*src == '\0')
3801 break;
3802
3803 if (*src == '\\')
3804 {
3805 src++;
3806 if (*src == 'n')
3807 dest[pos] = '\n';
3808 else
3809 dest[pos] = *src;
3810
3811 src++;
3812 continue;
3813 }
3814
3815 if (*src == '\0')
3816 break;
3817
3818 if (*src == '"')
3819 {
3820 src++;
3821 break;
3822 }
3823
3824 dest[pos] = *(src++);
3825 }
3826
3827 str_setlen(dest, pos);
3828
3829 if (!strcasecmp(tagstr, current_tag))
3830 return true;
3831 }
3832
3833 return false;
3834}
3835
3836static int parse_changelog_line(int line_n, char *buf, void *parameters)
3837{
3838 struct index_entry idx;
3839 char tag_data[TAGCACHE_BUFSZ];
3840 int idx_id;
3841 long masterfd = (long)(intptr_t)parameters;
3842 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime,
3843 tag_lastplayed, tag_commitid, tag_lastelapsed,
3844 tag_lastoffset };
3845 int i;
3846 (void)line_n;
3847
3848 if (*buf == '#')
3849 return 0;
3850
3851 /* logf("%d/%s", line_n, buf); */
3852 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3853 {
3854 logf("%d/filename missing", line_n);
3855 logf("-> %s", buf);
3856 return 0;
3857 }
3858
3859 idx_id = find_index(tag_data);
3860 if (idx_id < 0)
3861 {
3862 logf("%d/entry not found", line_n);
3863 return 0;
3864 }
3865
3866 if (!get_index(masterfd, idx_id, &idx, false))
3867 {
3868 logf("%d/failed to retrieve index entry", line_n);
3869 return 0;
3870 }
3871
3872 /* Stop if tag has already been modified. */
3873 if (idx.flag & FLAG_DIRTYNUM)
3874 return 0;
3875
3876 logf("%d/import: %s", line_n, tag_data);
3877
3878 idx.flag |= FLAG_DIRTYNUM;
3879 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3880 {
3881 int data;
3882
3883 if (!read_tag(tag_data, sizeof tag_data, buf,
3884 tagcache_tag_to_str(import_tags[i])))
3885 {
3886 continue;
3887 }
3888
3889 data = atoi(tag_data);
3890 if (data < 0)
3891 continue;
3892
3893 idx.tag_seek[import_tags[i]] = data;
3894
3895 if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3896 current_tcmh.serial = data + 1;
3897 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3898 current_tcmh.commitid = data + 1;
3899 }
3900
3901 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3902}
3903
3904bool tagcache_import_changelog(void)
3905{
3906 struct master_header myhdr;
3907 struct tagcache_header tch;
3908 int clfd;
3909 long masterfd;
3910 char buf[2048];
3911 const int bufsz = sizeof(buf);
3912
3913 if (!tc_stat.ready)
3914 return false;
3915
3916 while (read_lock)
3917 sleep(1);
3918
3919 clfd = open_db_fd(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3920 if (clfd < 0)
3921 {
3922 logf("failure to open changelog");
3923 return false;
3924 }
3925
3926 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3927 {
3928 close(clfd);
3929 return false;
3930 }
3931
3932 write_lock++;
3933
3934 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3935
3936 fast_readline(clfd, buf, bufsz, (void *)(intptr_t)masterfd,
3937 parse_changelog_line);
3938
3939 close(clfd);
3940 close(masterfd);
3941
3942 if (filenametag_fd >= 0)
3943 {
3944 close(filenametag_fd);
3945 filenametag_fd = -1;
3946 }
3947
3948 write_lock--;
3949
3950 update_master_header();
3951
3952 return true;
3953}
3954
3955#endif /* !__PCTOOL__ */
3956
3957bool tagcache_create_changelog(struct tagcache_search *tcs)
3958{
3959 struct master_header myhdr;
3960 struct index_entry idx;
3961 char buf[TAGCACHE_BUFSZ];
3962 const int bufsz = sizeof(buf);
3963 char temp[32];
3964 int clfd;
3965 int i, j;
3966
3967 if (!tc_stat.ready)
3968 return false;
3969
3970 if (!tagcache_search(tcs, tag_filename))
3971 return false;
3972
3973 /* Initialize the changelog */
3974 clfd = open_db_fd(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3975 if (clfd < 0)
3976 {
3977 logf("failure to open changelog");
3978 tagcache_search_finish(tcs);
3979 return false;
3980 }
3981
3982 if (tcs->masterfd < 0)
3983 {
3984 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3985 {
3986 close(clfd);
3987 tagcache_search_finish(tcs);
3988 return false;
3989 }
3990 }
3991 else
3992 {
3993 lseek(tcs->masterfd, 0, SEEK_SET);
3994 read_master_header(tcs->masterfd, &myhdr);
3995 }
3996
3997 write(clfd, "## Changelog version 1\n", 23);
3998
3999 for (i = 0; i < myhdr.tch.entry_count; i++)
4000 {
4001 if (read_index_entries(tcs->masterfd, &idx, 1) != sizeof(struct index_entry))
4002 {
4003 logf("read error #9");
4004 tagcache_search_finish(tcs);
4005 close(clfd);
4006 return false;
4007 }
4008
4009 /* Skip until the entry found has been modified. */
4010 if (! (idx.flag & FLAG_DIRTYNUM) )
4011 continue;
4012
4013 /* Skip deleted entries too. */
4014 if (idx.flag & FLAG_DELETED)
4015 continue;
4016
4017 /* Now retrieve all tags. */
4018 for (j = 0; j < TAG_COUNT; j++)
4019 {
4020 if (TAGCACHE_IS_NUMERIC(j))
4021 {
4022 itoa_buf(temp, sizeof temp, (int)idx.tag_seek[j]);
4023 write_tag(clfd, tagcache_tag_to_str(j), temp);
4024 continue;
4025 }
4026
4027 tcs->type = j;
4028 tagcache_retrieve(tcs, i, tcs->type, buf, bufsz);
4029 write_tag(clfd, tagcache_tag_to_str(j), buf);
4030 }
4031
4032 write(clfd, "\n", 1);
4033 do_timed_yield();
4034 }
4035
4036 close(clfd);
4037
4038 tagcache_search_finish(tcs);
4039
4040 return true;
4041}
4042
4043static bool delete_entry(long idx_id)
4044{
4045 int fd = -1;
4046 int masterfd = -1;
4047 int tag, i;
4048 struct index_entry idx, myidx;
4049 struct master_header myhdr;
4050 int in_use[TAG_COUNT];
4051
4052 logf("delete_entry(): %ld", idx_id);
4053
4054#ifdef HAVE_TC_RAMCACHE
4055 /* At first mark the entry removed from ram cache. */
4056 if (tc_stat.ramcache)
4057 tcramcache.hdr->indices[idx_id].flag |= FLAG_DELETED;
4058#endif
4059
4060 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
4061 return false;
4062
4063 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
4064 if (read_index_entries(masterfd, &myidx, 1) != sizeof(struct index_entry))
4065 {
4066 logf("delete_entry(): read error");
4067 goto cleanup;
4068 }
4069
4070 if (myidx.flag & FLAG_DELETED)
4071 {
4072 logf("delete_entry(): already deleted!");
4073 goto cleanup;
4074 }
4075
4076 myidx.flag |= FLAG_DELETED;
4077 lseek(masterfd, -(off_t)sizeof(struct index_entry), SEEK_CUR);
4078 if (write_index_entries(masterfd, &myidx, 1) != sizeof(struct index_entry))
4079 {
4080 logf("delete_entry(): write_error #1");
4081 goto cleanup;
4082 }
4083
4084 /* Now check which tags are no longer in use (if any) */
4085 for (tag = 0; tag < TAG_COUNT; tag++)
4086 in_use[tag] = 0;
4087
4088 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
4089 for (i = 0; i < myhdr.tch.entry_count; i++)
4090 {
4091 struct index_entry *idxp;
4092
4093#ifdef HAVE_TC_RAMCACHE
4094 /* Use RAM DB if available for greater speed */
4095 if (tc_stat.ramcache)
4096 idxp = &tcramcache.hdr->indices[i];
4097 else
4098#endif
4099 {
4100 if (read_index_entries(masterfd, &idx, 1) != sizeof(struct index_entry))
4101 {
4102 logf("delete_entry(): read error #2");
4103 goto cleanup;
4104 }
4105 idxp = &idx;
4106 }
4107
4108 if (idxp->flag & FLAG_DELETED)
4109 continue;
4110
4111 for (tag = 0; tag < TAG_COUNT; tag++)
4112 {
4113 if (TAGCACHE_IS_NUMERIC(tag))
4114 continue;
4115
4116 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
4117 in_use[tag]++;
4118 }
4119 }
4120
4121 /* Now delete all tags no longer in use. */
4122 for (tag = 0; tag < TAG_COUNT; tag++)
4123 {
4124 struct tagcache_header tch;
4125 int oldseek = myidx.tag_seek[tag];
4126
4127 if (TAGCACHE_IS_NUMERIC(tag))
4128 continue;
4129
4130 /**
4131 * Replace tag seek with a hash value of the field string data.
4132 * That way runtime statistics of moved or altered files can be
4133 * resurrected.
4134 */
4135#ifdef HAVE_TC_RAMCACHE
4136 if (tc_stat.ramcache && tag != tag_filename)
4137 {
4138 struct tagfile_entry *tfe;
4139 int32_t *seek = &tcramcache.hdr->indices[idx_id].tag_seek[tag];
4140
4141 /* crc_32 is assumed not to yield (why would it...?) */
4142 tfe = (struct tagfile_entry *)&tcramcache.hdr->tags[tag][*seek];
4143 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
4144 myidx.tag_seek[tag] = *seek;
4145 }
4146 else
4147#endif /* HAVE_TC_RAMCACHE */
4148 {
4149 struct tagfile_entry tfe;
4150
4151 /* Open the index file, which contains the tag names. */
4152 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
4153 goto cleanup;
4154
4155 /* Skip the header block */
4156 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
4157
4158 switch (read_tagfile_entry_and_tag(fd, &tfe,
4159 build_idx_buf, build_idx_bufsz))
4160 {
4161 case e_SUCCESS_LEN_ZERO:
4162 logf("deleted_entry(): SUCCESS");
4163 /* FALL THROUGH */
4164 case e_SUCCESS:
4165 break;
4166 case e_ENTRY_SIZEMISMATCH:
4167 logf("delete_entry(): read error #3");
4168 goto cleanup;
4169 case e_TAG_TOOLONG:
4170 logf("too long tag #4");
4171 goto cleanup;
4172 case e_TAG_SIZEMISMATCH:
4173 logf("delete_entry(): read error #3");
4174 goto cleanup;
4175 }
4176
4177 myidx.tag_seek[tag] = crc_32(build_idx_buf,
4178 strlen(build_idx_buf), 0xffffffff);
4179 }
4180
4181 if (in_use[tag])
4182 {
4183 logf("in use: %d/%d", tag, in_use[tag]);
4184 if (fd >= 0)
4185 {
4186 close(fd);
4187 fd = -1;
4188 }
4189 continue;
4190 }
4191
4192#ifdef HAVE_TC_RAMCACHE
4193 /* Delete from ram. */
4194 if (tc_stat.ramcache && tag != tag_filename)
4195 {
4196 struct tagfile_entry *tagentry =
4197 (struct tagfile_entry *)&tcramcache.hdr->tags[tag][oldseek];
4198 str_setlen(tagentry->tag_data, 0);
4199 }
4200#endif /* HAVE_TC_RAMCACHE */
4201
4202 /* Open the index file, which contains the tag names. */
4203 if (fd < 0)
4204 {
4205 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
4206 goto cleanup;
4207 }
4208
4209 /* Skip the header block */
4210 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
4211
4212 /* Debug, print 10 first characters of the tag
4213 read(fd, buf, 10);
4214 buf[10]='\0';
4215 logf("TAG:%s", buf);
4216 lseek(fd, -10, SEEK_CUR);
4217 */
4218
4219 /* Write first data byte in tag as \0 */
4220 write(fd, "", 1);
4221
4222 /* Now tag data has been removed */
4223 close(fd);
4224 fd = -1;
4225 }
4226
4227 /* Write index entry back into master index. */
4228 lseek(masterfd, sizeof(struct master_header) +
4229 (idx_id * sizeof(struct index_entry)), SEEK_SET);
4230 if (write_index_entries(masterfd, &myidx, 1) != sizeof(struct index_entry))
4231 {
4232 logf("delete_entry(): write_error #2");
4233 goto cleanup;
4234 }
4235
4236 close(masterfd);
4237
4238 return true;
4239
4240 cleanup:
4241 if (fd >= 0)
4242 close(fd);
4243 if (masterfd >= 0)
4244 close(masterfd);
4245
4246 return false;
4247}
4248
4249/**
4250 * Returns true if there is an event waiting in the queue
4251 * that requires the current operation to be aborted.
4252 */
4253static bool check_event_queue(void)
4254{
4255#ifndef __PCTOOL__
4256 struct queue_event ev;
4257
4258 if(!queue_peek(&tagcache_queue, &ev))
4259 return false;
4260
4261 switch (ev.id)
4262 {
4263 case Q_STOP_SCAN:
4264 case SYS_POWEROFF:
4265 case SYS_REBOOT:
4266 case SYS_USB_CONNECTED:
4267 return true;
4268 }
4269#endif /* __PCTOOL__ */
4270
4271 return false;
4272}
4273
4274#ifdef HAVE_TC_RAMCACHE
4275
4276static void fix_ramcache(void* old_addr, void* new_addr)
4277{
4278 ptrdiff_t offpos = new_addr - old_addr;
4279 for (int i = 0; i < TAG_COUNT; i++)
4280 tcramcache.hdr->tags[i] += offpos;
4281}
4282
4283static int move_cb(int handle, void* current, void* new)
4284{
4285 (void)handle;
4286 fix_ramcache(current, new);
4287 tcramcache.hdr = new;
4288 return BUFLIB_CB_OK;
4289}
4290
4291static struct buflib_callbacks ops = {
4292 .move_callback = move_cb,
4293 .shrink_callback = NULL,
4294};
4295
4296static bool allocate_tagcache(void)
4297{
4298 tc_stat.ramcache_allocated = 0;
4299 tcramcache.handle = 0;
4300 tcramcache.hdr = NULL;
4301
4302 /* Load the header. */
4303 struct master_header tcmh;
4304 int fd = open_master_fd(&tcmh, false);
4305 if (fd < 0)
4306 return false;
4307
4308 close(fd);
4309
4310 /**
4311 * Now calculate the required cache size plus
4312 * some extra space for alignment fixes.
4313 */
4314 size_t alloc_size = tcmh.tch.datasize + 256 + TAGCACHE_RESERVE +
4315 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
4316#ifdef HAVE_DIRCACHE
4317 alloc_size += tcmh.tch.entry_count*sizeof(struct dircache_fileref);
4318#endif
4319
4320 int handle = core_alloc_ex(alloc_size, &ops);
4321 if (handle <= 0)
4322 return false;
4323
4324 tcramcache.handle = handle;
4325 tcramcache.hdr = core_get_data(handle);
4326 tc_stat.ramcache_allocated = alloc_size;
4327
4328 memset(tcramcache.hdr, 0, sizeof(struct ramcache_header));
4329 memcpy(¤t_tcmh, &tcmh, sizeof current_tcmh);
4330 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
4331
4332 return true;
4333}
4334
4335#ifdef HAVE_EEPROM_SETTINGS
4336static bool tagcache_dumpload(void)
4337{
4338 struct statefile_header shdr;
4339 int fd, rc, handle;
4340
4341 tcramcache.handle = 0;
4342 tcramcache.hdr = NULL;
4343
4344 fd = open_db_fd(TAGCACHE_STATEFILE, O_RDONLY);
4345 if (fd < 0)
4346 {
4347 logf("no tagcache statedump");
4348 return false;
4349 }
4350
4351 /* Check the statefile memory placement */
4352 rc = read(fd, &shdr, sizeof(struct statefile_header));
4353 if (rc != sizeof(struct statefile_header)
4354 || shdr.magic != TAGCACHE_STATEFILE_MAGIC
4355 || shdr.mh.tch.magic != TAGCACHE_MAGIC)
4356 {
4357 logf("incorrect statefile");
4358 close(fd);
4359 return false;
4360 }
4361
4362 /* Lets allocate real memory and load it */
4363 handle = core_alloc_ex(shdr.tc_stat.ramcache_allocated, &ops);
4364 if (handle <= 0)
4365 {
4366 logf("alloc failure");
4367 return false;
4368 }
4369
4370 tcramcache.handle = handle;
4371 tcrc_buffer_lock();
4372 tcramcache.hdr = core_get_data(handle);
4373 rc = read(fd, tcramcache.hdr, shdr.tc_stat.ramcache_allocated);
4374 tcrc_buffer_unlock();
4375
4376 close(fd);
4377
4378 if (rc != shdr.tc_stat.ramcache_allocated)
4379 {
4380 logf("read failure!");
4381 core_free(handle);
4382 return false;
4383 }
4384
4385 tc_stat = shdr.tc_stat;
4386
4387 /* Now fix the pointers */
4388 fix_ramcache(shdr.hdr, tcramcache.hdr);
4389
4390 /* Load the tagcache master header (should match the actual DB file header). */
4391 memcpy(¤t_tcmh, &shdr.mh, sizeof current_tcmh);
4392
4393 return true;
4394}
4395
4396static bool tagcache_dumpsave(void)
4397{
4398 struct statefile_header shdr;
4399 int fd;
4400
4401 if (!tc_stat.ramcache)
4402 return false;
4403
4404 fd = open_db_fd(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
4405 if (fd < 0)
4406 {
4407 logf("failed to create a statedump");
4408 return false;
4409 }
4410
4411 /* Create the header */
4412 shdr.magic = TAGCACHE_STATEFILE_MAGIC;
4413 shdr.hdr = tcramcache.hdr;
4414 memcpy(&shdr.mh, ¤t_tcmh, sizeof current_tcmh);
4415 memcpy(&shdr.tc_stat, &tc_stat, sizeof tc_stat);
4416 write(fd, &shdr, sizeof shdr);
4417
4418 /* And dump the data too */
4419 tcrc_buffer_lock();
4420 write(fd, tcramcache.hdr, tc_stat.ramcache_allocated);
4421 tcrc_buffer_unlock();
4422 close(fd);
4423
4424 return true;
4425}
4426#endif /* HAVE_EEPROM_SETTINGS */
4427
4428static bool load_tagcache(void)
4429{
4430 /* DEBUG: After tagcache commit and dircache rebuild, hdr-sturcture
4431 * may become corrupt. */
4432
4433 bool ok = false;
4434 ssize_t bytesleft = tc_stat.ramcache_allocated - sizeof(struct ramcache_header);
4435 int fd;
4436
4437#ifdef HAVE_DIRCACHE
4438 /* Wait for any in-progress dircache build to complete */
4439 dircache_wait();
4440#endif /* HAVE_DIRCACHE */
4441
4442 logf("loading tagcache to ram...");
4443
4444 tcrc_buffer_lock(); /* lock for the rest of the scan, simpler to handle */
4445
4446 fd = open_db_fd(TAGCACHE_FILE_MASTER, O_RDONLY);
4447 if (fd < 0)
4448 {
4449 logf("tagcache open failed");
4450 goto failure;
4451 }
4452
4453 struct master_header tcmh;
4454 if (read_master_header(fd, &tcmh) != sizeof(struct master_header) ||
4455 tcmh.tch.magic != TAGCACHE_MAGIC)
4456 {
4457 logf("incorrect header");
4458 goto failure;
4459 }
4460
4461 /* Master header copy should already match, this can be redundant to do. */
4462 current_tcmh = tcmh;
4463
4464 /* Load the master index table. */
4465 for (int i = 0; i < tcmh.tch.entry_count; i++)
4466 {
4467 bytesleft -= sizeof(struct index_entry);
4468 if (bytesleft < 0)
4469 {
4470 logf("too big tagcache.");
4471 goto failure;
4472 }
4473
4474 int rc = read_index_entries(fd, &tcramcache.hdr->indices[i], 1);
4475 if (rc != sizeof (struct index_entry))
4476 {
4477 logf("read error #10");
4478 goto failure;
4479 }
4480 }
4481
4482 close(fd);
4483 fd = -1;
4484
4485 /* Load the tags right after the index entries */
4486 char *p = (char *)&tcramcache.hdr->indices[tcmh.tch.entry_count];
4487
4488 for (int tag = 0; tag < TAG_COUNT; tag++)
4489 {
4490 ssize_t rc;
4491
4492 if (TAGCACHE_IS_NUMERIC(tag))
4493 continue;
4494
4495 p = TC_ALIGN_PTR(p, struct tagcache_header, &rc);
4496 bytesleft -= rc;
4497 if (bytesleft < (ssize_t)sizeof(struct tagcache_header))
4498 {
4499 logf("Too big tagcache #10.5");
4500 goto failure;
4501 }
4502
4503 tcramcache.hdr->tags[tag] = p;
4504
4505 /* Load the header */
4506 struct tagcache_header *tch = (struct tagcache_header *)p;
4507 p += sizeof(struct tagcache_header);
4508 bytesleft -= sizeof (struct tagcache_header);
4509
4510 fd = open_tag_fd(tch, tag, false);
4511 if (rc < 0)
4512 goto failure;
4513
4514 /* Load the entries for this tag */
4515 for (tcramcache.hdr->entry_count[tag] = 0;
4516 tcramcache.hdr->entry_count[tag] < tch->entry_count;
4517 tcramcache.hdr->entry_count[tag]++)
4518 {
4519 /* Abort if we got a critical event in queue */
4520 if (do_timed_yield() && check_event_queue())
4521 goto failure;
4522
4523 p = TC_ALIGN_PTR(p, struct tagfile_entry, &rc);
4524 bytesleft -= rc;
4525 if (bytesleft < (ssize_t)sizeof(struct tagfile_entry))
4526 {
4527 logf("Too big tagcache #10.75");
4528 goto failure;
4529 }
4530
4531 struct tagfile_entry *fe = (struct tagfile_entry *)p;
4532 off_t pos = lseek(fd, 0, SEEK_CUR);
4533
4534 /* Load the header for the tag itself */
4535 if (read_tagfile_entry(fd, fe) != sizeof(struct tagfile_entry))
4536 {
4537 /* End of lookup table. */
4538 logf("read error #11");
4539 goto failure;
4540 }
4541
4542 int idx_id = fe->idx_id; /* dircache reference clobbers *fe */
4543 struct index_entry *idx = &tcramcache.hdr->indices[idx_id];
4544
4545 if (idx_id != -1 || tag == tag_filename) /* filename NOT optional */
4546 {
4547 if (idx_id < 0 || idx_id >= tcmh.tch.entry_count)
4548 {
4549 logf("corrupt tagfile entry:tag=%d:idxid=%d", tag, idx_id);
4550 goto failure;
4551 }
4552
4553 if (idx->tag_seek[tag] != pos)
4554 {
4555 logf("corrupt data structures!:");
4556 logf(" tag_seek[%d]=%" PRId32 ":pos=%ld", tag,
4557 idx->tag_seek[tag], (long) pos);
4558 goto failure;
4559 }
4560 }
4561
4562 /* We have a special handling for the filename tags; neither the
4563 paths nor the entry headers are stored; only the tagcache header
4564 and dircache references are. */
4565 if (tag == tag_filename)
4566 {
4567 #ifdef HAVE_DIRCACHE
4568 if (idx->flag & FLAG_DIRCACHE)
4569 {
4570 /* This flag must not be used yet. */
4571 logf("internal error!");
4572 goto failure;
4573 }
4574
4575 p += sizeof (struct dircache_fileref);
4576 bytesleft -= sizeof (struct dircache_fileref);
4577 #endif /* HAVE_DIRCACHE */
4578
4579 char filename[TAGCACHE_BUFSZ];
4580 if (fe->tag_length >= (long)sizeof(filename)-1)
4581 {
4582 read(fd, filename, 10);
4583 str_setlen(filename, 10);
4584 logf("TAG:%s", filename);
4585 logf("too long filename");
4586 goto failure;
4587 }
4588
4589 if ((idx->flag & FLAG_DELETED)
4590 IFN_DIRCACHE( || !global_settings.tagcache_autoupdate ))
4591 {
4592 /* seek over tag data instead of reading */
4593 if (lseek(fd, fe->tag_length, SEEK_CUR) < 0)
4594 {
4595 logf("read error #11.5");
4596 goto failure;
4597 }
4598
4599 continue;
4600 }
4601
4602 if (read(fd, filename, fe->tag_length) != fe->tag_length)
4603 {
4604 logf("read error #12");
4605 goto failure;
4606 }
4607 continue;
4608 }
4609
4610 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4611 if (bytesleft < 0)
4612 {
4613 logf("too big tagcache #2");
4614 logf("tl: %" PRId32, fe->tag_length);
4615 logf("bl: %ld", (long) bytesleft);
4616 goto failure;
4617 }
4618
4619 p = fe->tag_data;
4620 rc = read(fd, p, fe->tag_length);
4621 p += rc;
4622
4623 if (rc != fe->tag_length)
4624 {
4625 logf("read error #13");
4626 logf("rc=0x%04x", (unsigned int)rc); // 0x431
4627 logf("len=0x%04" PRIx32, fe->tag_length); // 0x4000
4628 logf("pos=0x%04lx", (unsigned long) lseek(fd, 0, SEEK_CUR)); // 0x433
4629 logf("tag=0x%02x", tag); // 0x00
4630 goto failure;
4631 }
4632 }
4633
4634 #ifdef HAVE_DIRCACHE
4635 if (tag == tag_filename)
4636 p = (char *)&tcrc_dcfrefs[tcmh.tch.entry_count];
4637 #endif /* HAVE_DIRCACHE */
4638
4639 close(fd);
4640 }
4641
4642 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4643 logf("tagcache loaded into ram!");
4644 logf("utilization: %d%%", 100*tc_stat.ramcache_used / tc_stat.ramcache_allocated);
4645
4646 ok = true;
4647
4648failure:
4649 if (fd >= 0)
4650 close(fd);
4651
4652 tcrc_buffer_unlock();
4653 return ok;
4654}
4655#endif /* HAVE_TC_RAMCACHE */
4656
4657static bool check_file_refs(bool auto_update)
4658{
4659 int fd;
4660 bool ret = true;
4661 char buf[TAGCACHE_BUFSZ];
4662 const int bufsz = sizeof(buf);
4663 struct tagfile_entry tfe;
4664 struct tagcache_header hdr;
4665
4666 logf("reverse scan...");
4667
4668#ifdef HAVE_DIRCACHE
4669 if (tcramcache.handle > 0)
4670 tcrc_buffer_lock();
4671 else
4672 return false;
4673 /* Wait for any in-progress dircache build to complete */
4674 dircache_wait();
4675#else
4676 if (!auto_update)
4677 return false;
4678#endif
4679
4680 fd = open_tag_fd(&hdr, tag_filename, false); /* open read only*/
4681
4682 if (fd < 0)
4683 {
4684 logf(TAGCACHE_FILE_INDEX " open fail", tag_filename);
4685 return false;
4686 }
4687
4688 processed_dir_count = 0;
4689
4690 while (!check_event_queue())
4691 {
4692 int res = read_tagfile_entry_and_tag(fd, &tfe, buf, bufsz);
4693 processed_dir_count++;
4694
4695 switch (res)
4696 {
4697 case e_ENTRY_SIZEMISMATCH:
4698 logf("size mismatch entry EOF?"); /* likely EOF */
4699 ret = false;
4700 goto wend_finished;
4701 case e_TAG_TOOLONG:
4702 logf("too long tag");
4703 ret = false;
4704 goto wend_finished;
4705 case e_TAG_SIZEMISMATCH:
4706 logf("size mismatch tag - read error #14");
4707 ret = false;
4708 goto wend_finished;
4709 case e_SUCCESS:
4710 break;
4711 case e_SUCCESS_LEN_ZERO:
4712 continue;
4713 }
4714
4715 int idx_id = tfe.idx_id; /* dircache reference clobbers *tfe */
4716#ifdef HAVE_DIRCACHE
4717 struct index_entry *idx = &tcramcache.hdr->indices[idx_id];
4718 unsigned int searchflag;
4719 if (!auto_update)
4720 {
4721 if(idx->flag & FLAG_DIRCACHE) /* already found */
4722 {
4723 continue;
4724 }
4725 searchflag = DCS_CACHED_PATH; /* attempt to load cache references */
4726 }
4727 else /* If auto updating, check storage too */
4728 {
4729 searchflag = DCS_STORAGE_PATH;
4730 }
4731
4732 int rc_cache = dircache_search(searchflag | DCS_UPDATE_FILEREF,
4733 &tcrc_dcfrefs[idx_id], buf);
4734
4735 if (rc_cache > 0) /* in cache and we have fileref */
4736 {
4737 idx->flag |= FLAG_DIRCACHE;
4738 }
4739 else if (rc_cache == 0) /* not in cache but okay */
4740 {;}
4741 else if (auto_update && rc_cache == ENOENT)
4742#else
4743 if (!file_exists(buf))
4744#endif /* HAVE_DIRCACHE */
4745 {
4746 logf("Entry no longer valid.");
4747 logf("-> %s / %" PRId32, buf, tfe.tag_length);
4748 delete_entry(idx_id);
4749 }
4750
4751 do_timed_yield();
4752 }
4753
4754wend_finished:
4755
4756#ifdef HAVE_DIRCACHE
4757 if (tcramcache.handle > 0)
4758 tcrc_buffer_unlock();
4759#endif
4760 close(fd);
4761 logf("done");
4762
4763 return ret;
4764}
4765
4766static bool check_deleted_files(void)
4767{
4768 return check_file_refs(true);
4769}
4770
4771/* Note that this function must not be inlined, otherwise the whole point
4772 * of having the code in a separate function is lost.
4773 */
4774static void NO_INLINE check_ignore(const char *dirname,
4775 int *ignore, int *unignore)
4776{
4777 char newpath[MAX_PATH];
4778 const int bufsz = sizeof(newpath);
4779
4780 /* check for a database.ignore file */
4781 snprintf(newpath, bufsz, "%s/database.ignore", dirname);
4782 *ignore = file_exists(newpath);
4783 /* check for a database.unignore file */
4784 snprintf(newpath, bufsz, "%s/database.unignore", dirname);
4785 *unignore = file_exists(newpath);
4786}
4787
4788/* max roots on native. on application more can be added via malloc() */
4789#define MAX_STATIC_ROOTS 12
4790
4791static struct search_roots_ll {
4792 const char *path;
4793 struct search_roots_ll * next;
4794} roots_ll[MAX_STATIC_ROOTS];
4795
4796/* check if the path is already included in the search roots, by the
4797 * means that the path itself or one of its parents folders is in the list */
4798static bool search_root_exists(const char *path)
4799{
4800 struct search_roots_ll *this;
4801 for(this = &roots_ll[0]; this; this = this->next)
4802 {
4803 size_t root_len = strlen(this->path);
4804 /* check if the link target is inside of an existing search root
4805 * don't add if target is inside, we'll scan it later */
4806 if (!strncmp(this->path, path, root_len))
4807 return true;
4808 }
4809 return false;
4810}
4811
4812#ifdef APPLICATION
4813/*
4814 * This adds a path to the search roots, possibly during traveling through
4815 * the filesystem. It only adds if the path is not inside an already existing
4816 * search root.
4817 *
4818 * Returns true if it added the path to the search roots
4819 *
4820 * Windows 2000 and greater supports symlinks, but they don't provide
4821 * realpath() or readlink(), and symlinks are rarely used on them so
4822 * ignore this for windows for now
4823 **/
4824static bool add_search_root(const char *name)
4825{
4826 (void)name;
4827#ifndef WIN32
4828 struct search_roots_ll *this, *prev = NULL;
4829 char target[MAX_PATH];
4830 const int target_bufsz = sizeof(target);
4831 /* Okay, realpath() is almost completely broken on android
4832 *
4833 * It doesn't accept NULL for resolved_name to dynamically allocate
4834 * the resulting path; and it assumes resolved_name to be PATH_MAX
4835 * (actually MAXPATHLEN, but it's the same [as of 2.3]) long
4836 * and blindly writes to the end if it
4837 *
4838 * therefore use sufficiently large static storage here
4839 * Note that PATH_MAX != MAX_PATH
4840 **/
4841 static char abs_target[PATH_MAX];
4842 ssize_t len;
4843
4844 len = readlink(name, target, target_bufsz-1);
4845 if (len < 0)
4846 return false;
4847
4848 str_setlen(target, len);
4849 if (realpath(target, abs_target) == NULL)
4850 return false;
4851
4852 if (search_root_exists(abs_target))
4853 return false;
4854
4855 /* get the end of the list */
4856 for(this = &roots_ll[0]; this; prev = this, this = this->next);
4857
4858 if (prev)
4859 {
4860 size_t len = strlen(abs_target) + 1; /* count \0 */
4861 this = malloc(sizeof(struct search_roots_ll) + len );
4862 if (!this || len > MIN(PATH_MAX, MAX_PATH))
4863 {
4864 logf("Error at adding a search root: %s", this ? "path too long":"OOM");
4865 free(this);
4866 prev->next = NULL;
4867 return false;
4868 }
4869 this->path = ((char*)this) + sizeof(struct search_roots_ll);
4870 strcpy((char*)this->path, abs_target); /* ok to cast const away here */
4871 this->next = NULL;
4872 prev->next = this;
4873 logf("Added %s to the search roots\n", abs_target);
4874 return true;
4875 }
4876#endif
4877 return false;
4878}
4879
4880static int free_search_root_single(struct search_roots_ll * start)
4881{
4882 if (start < &roots_ll[0] && start >= &roots_ll[MAX_STATIC_ROOTS])
4883 {
4884 free(start->next);
4885 return sizeof(struct search_roots_ll);
4886 }
4887 return 0;
4888}
4889
4890static int free_search_roots(struct search_roots_ll * start)
4891{
4892 int ret = 0;
4893 if (start->next)
4894 {
4895 ret += free_search_root_single(start->next);
4896 }
4897 return ret;
4898}
4899#else /* native, simulator */
4900#define add_search_root(a) do {} while(0)
4901#define free_search_roots(a) do {} while(0)
4902#endif
4903
4904static bool check_dir(const char *dirname, int add_files)
4905{
4906 int success = false;
4907
4908 DIR *dir = opendir(dirname);
4909 if (!dir)
4910 {
4911 logf("tagcache: opendir(%s) failed", dirname);
4912 return false;
4913 }
4914
4915 /* check for a database.ignore and database.unignore */
4916 int ignore, unignore;
4917 check_ignore(dirname, &ignore, &unignore);
4918
4919 /* don't do anything if both ignore and unignore are there */
4920 if (ignore != unignore)
4921 add_files = unignore;
4922
4923 /* Recursively scan the dir. */
4924 while (!check_event_queue())
4925 {
4926 struct dirent *entry = readdir(dir);
4927 if (entry == NULL)
4928 {
4929 success = true;
4930 break;
4931 }
4932
4933 if (is_dotdir_name(entry->d_name))
4934 continue;
4935
4936 struct dirinfo info = dir_get_info(dir, entry);
4937 size_t len = strlen(curpath);
4938 path_append(&curpath[len-1], PA_SEP_HARD, entry->d_name,
4939 sizeof (curpath) - len);
4940
4941 processed_dir_count++;
4942 if (info.attribute & ATTR_DIRECTORY)
4943 {
4944#ifndef SIMULATOR
4945 /* don't follow symlinks to dirs, but try to add it as a search root
4946 * this makes able to avoid looping in recursive symlinks */
4947 if (info.attribute & ATTR_LINK)
4948 add_search_root(curpath);
4949 else
4950#endif /* SIMULATOR */
4951 check_dir(curpath, add_files);
4952 }
4953 else if (add_files)
4954 {
4955 tc_stat.curentry = curpath;
4956
4957 /* Add a new entry to the temporary db file. */
4958 add_tagcache(curpath, info.mtime);
4959
4960 /* Wait until current path for debug screen is read and unset. */
4961 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4962 yield();
4963
4964 tc_stat.curentry = NULL;
4965 }
4966
4967 str_setlen(curpath, len);
4968 }
4969
4970 closedir(dir);
4971
4972 return success;
4973}
4974
4975void tagcache_screensync_event(void)
4976{
4977 tc_stat.curentry = NULL;
4978}
4979
4980void tagcache_screensync_enable(bool state)
4981{
4982 tc_stat.syncscreen = state;
4983}
4984
4985#ifndef __PCTOOL__
4986/* this is called by the database tool to not pull in global_settings */
4987static
4988#endif
4989void do_tagcache_build(const char *path[])
4990{
4991 struct tagcache_header header;
4992 bool ret;
4993
4994 str_setlen(curpath, 0);
4995 data_size = 0;
4996 total_entry_count = 0;
4997 processed_dir_count = 0;
4998
4999#ifdef HAVE_DIRCACHE
5000 dircache_wait();
5001#endif
5002
5003 logf("updating tagcache");
5004
5005 cachefd = open_db_fd(TAGCACHE_FILE_TEMP, O_RDONLY);
5006 if (cachefd >= 0)
5007 {
5008 logf("skipping, cache already waiting for commit");
5009 close(cachefd);
5010 return ;
5011 }
5012
5013 cachefd = open_db_fd(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
5014 if (cachefd < 0)
5015 {
5016 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
5017 return ;
5018 }
5019
5020 filenametag_fd = open_tag_fd(&header, tag_filename, false);
5021
5022 cpu_boost(true);
5023
5024 logf("Scanning files...");
5025 /* Scan for new files. */
5026 memset(&header, 0, sizeof(struct tagcache_header));
5027 write(cachefd, &header, sizeof(struct tagcache_header));
5028
5029 ret = true;
5030
5031 roots_ll[0].path = path[0];
5032 roots_ll[0].next = NULL;
5033
5034#if defined(HAVE_MULTIVOLUME) && !defined(SIMULATOR) && !defined(__PCTOOL__) && !defined(APPLICATION)
5035 extern bool ns_volume_is_visible(int volume); /*rb_namespace.c*/
5036 /* i is for the path vector, j for the roots_ll array */
5037 int i = 1, j = 1;
5038 bool added = false;
5039 char volnamebuf[NUM_VOLUMES][VOL_MAX_LEN + 1];
5040 /* we can just parse the root directory ('/') and get to any mounted
5041 * volume but we can also enumerate a volume in the root directory
5042 * when this occurs it leads to multiple entries since the files can
5043 * be reached through multiple paths ex, /Foo could also be /SD1/Foo
5044 * we used to hide the volume that was mapped but then when you switch
5045 * from the sd to the internal the paths don't map to the right volume
5046 * instead we will attempt to rewrite the root with any non-hidden volumes
5047 * failing that just leave the paths alone */
5048 if (!strcmp(PATH_ROOTSTR, path[0]))
5049 {
5050 i = 0;
5051 j = 0;
5052 }
5053 /* path can be skipped , but root_ll entries can't */
5054 for(; path[i] && j < MAX_STATIC_ROOTS; i++)
5055 {
5056 /* check if the link target is inside of an existing search root
5057 * don't add if target is inside, we'll scan it later */
5058 if (!added && !strcmp(PATH_ROOTSTR, path[i]))
5059 {
5060 for (int v = 0; v < NUM_VOLUMES; v++)
5061 {
5062 if (ns_volume_is_visible(v))
5063 {
5064 make_volume_root(v, volnamebuf[v]);
5065 roots_ll[j].path = volnamebuf[v];
5066 if (j > 0)
5067 roots_ll[j-1].next = &roots_ll[j];
5068 j++;
5069 added = true;
5070 }
5071 }
5072 if(!added)
5073 j = 1;
5074 added = true;
5075 continue;
5076 }
5077#else
5078 /* i is for the path vector, j for the roots_ll array
5079 * path can be skipped , but root_ll entries can't */
5080 for(int i = 1, j = 1; path[i] && j < MAX_STATIC_ROOTS; i++)
5081 {
5082#endif /*def HAVE_MULTIVOLUME*/
5083 if (search_root_exists(path[i])) /* skip this path */
5084 continue;
5085
5086 roots_ll[j].path = path[i];
5087 roots_ll[j-1].next = &roots_ll[j];
5088 j++;
5089 }
5090
5091 struct search_roots_ll * this;
5092 /* check_dir might add new roots */
5093 for(this = &roots_ll[0]; this; this = this->next)
5094 {
5095 logf("Search root %s", this->path);
5096 strmemccpy(curpath, this->path, sizeof(curpath));
5097
5098 if (ret)
5099 {
5100 if (dir_exists(this->path))
5101 ret = check_dir(this->path, true);
5102 else
5103 logf("Dir not found %s", this->path);
5104 }
5105 }
5106 free_search_roots(&roots_ll[0]);
5107
5108 /* Write the header. */
5109 header.magic = TAGCACHE_MAGIC;
5110 header.datasize = data_size;
5111 header.entry_count = total_entry_count;
5112 lseek(cachefd, 0, SEEK_SET);
5113 write(cachefd, &header, sizeof(struct tagcache_header));
5114 close(cachefd);
5115
5116 if (filenametag_fd >= 0)
5117 {
5118 close(filenametag_fd);
5119 filenametag_fd = -1;
5120 }
5121
5122 if (!ret)
5123 {
5124 logf("Aborted.");
5125 cpu_boost(false);
5126 return ;
5127 }
5128
5129 /* Commit changes to the database. */
5130#ifdef __PCTOOL__
5131 allocate_tempbuf();
5132#endif
5133 if (commit())
5134 {
5135 logf("tagcache built!");
5136 }
5137#ifdef __PCTOOL__
5138 free_tempbuf();
5139#endif
5140
5141#ifdef HAVE_TC_RAMCACHE
5142 if (tcramcache.hdr)
5143 {
5144 /* Import runtime statistics if we just initialized the db. */
5145 if (current_tcmh.serial == 0)
5146 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
5147 }
5148#endif
5149
5150 cpu_boost(false);
5151}
5152
5153#ifndef __PCTOOL__
5154void tagcache_build(void)
5155{
5156 char *vect[MAX_STATIC_ROOTS + 1]; /* +1 to ensure NULL sentinel */
5157 char str[sizeof(global_settings.tagcache_scan_paths)];
5158 strmemccpy(str, global_settings.tagcache_scan_paths, sizeof(str));
5159
5160 int res = split_string(str, ':', vect, MAX_STATIC_ROOTS);
5161 vect[res] = NULL;
5162
5163 do_tagcache_build((const char**)vect);
5164}
5165#endif /* __PCTOOL__ */
5166
5167#ifdef HAVE_TC_RAMCACHE
5168static void load_ramcache(void)
5169{
5170 if (!tcramcache.hdr)
5171 return ;
5172
5173 cpu_boost(true);
5174
5175 /* At first we should load the cache (if exists). */
5176 tc_stat.ramcache = load_tagcache();
5177
5178 if (!tc_stat.ramcache)
5179 {
5180 /* If loading failed, it must indicate some problem with the db
5181 * so disable it entirely to prevent further issues. */
5182 tc_stat.ready = false;
5183 tcramcache.hdr = NULL;
5184 int handle = tcramcache.handle;
5185 tcramcache.handle = 0;
5186 core_free(handle);
5187 }
5188
5189 cpu_boost(false);
5190}
5191
5192void tagcache_unload_ramcache(void)
5193{
5194 tc_stat.ramcache = false;
5195 /* Just to make sure there is no statefile present. */
5196 // remove_db_file(TAGCACHE_STATEFILE);
5197}
5198#endif /* HAVE_TC_RAMCACHE */
5199
5200#ifndef __PCTOOL__
5201
5202/*
5203 * db_file_exists is noinline to minimize stack usage
5204 */
5205static bool NO_INLINE db_file_exists(const char* filename)
5206{
5207 char buf[MAX_PATH];
5208
5209 snprintf(buf, sizeof(buf), "%s/%s", tc_stat.db_path, filename);
5210
5211 return file_exists(buf);
5212}
5213
5214static void tagcache_thread(void)
5215{
5216 struct queue_event ev;
5217 bool check_done = false;
5218 cpu_boost(true);
5219 /* If the previous cache build/update was interrupted, commit
5220 * the changes first in foreground. */
5221 if (db_file_exists(TAGCACHE_FILE_TEMP))
5222 {
5223#if !(defined(__APPLE__) && (CONFIG_PLATFORM & PLATFORM_SDL))
5224 static const char *lines[] = {ID2P(LANG_TAGCACHE_BUSY),
5225 ID2P(LANG_TAGCACHE_UPDATE)};
5226 static const struct text_message message = {lines, 2};
5227
5228 if (gui_syncyesno_run_w_tmo(HZ * 5, YESNO_YES, &message, NULL, NULL) == YESNO_YES)
5229#endif
5230 {
5231 allocate_tempbuf();
5232 commit();
5233 free_tempbuf();
5234 }
5235 }
5236
5237#ifdef HAVE_TC_RAMCACHE
5238#ifdef HAVE_EEPROM_SETTINGS
5239 if (firmware_settings.initialized && firmware_settings.disk_clean
5240 && global_settings.tagcache_ram)
5241 {
5242 check_done = tagcache_dumpload();
5243 }
5244
5245 remove_db_file(TAGCACHE_STATEFILE);
5246#endif /* HAVE_EEPROM_SETTINGS */
5247
5248 /* Allocate space for the tagcache if found on disk. */
5249 if (global_settings.tagcache_ram && !tc_stat.ramcache)
5250 allocate_tagcache();
5251#endif /* HAVE_TC_RAMCACHE */
5252
5253 cpu_boost(false);
5254 tc_stat.initialized = true;
5255
5256 /* Don't delay bootup with the header check but do it on background. */
5257 if (!tc_stat.ready)
5258 {
5259 sleep(HZ);
5260 tagcache_commit_finalize();
5261 }
5262
5263 while (1)
5264 {
5265 run_command_queue(false);
5266
5267 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
5268
5269 switch (ev.id)
5270 {
5271 case Q_IMPORT_CHANGELOG:
5272 tagcache_import_changelog();
5273 break;
5274
5275 case Q_REBUILD:
5276 remove_files();
5277 remove_db_file(TAGCACHE_FILE_TEMP);
5278 tagcache_build();
5279 break;
5280
5281 case Q_UPDATE:
5282 tagcache_build();
5283#ifdef HAVE_TC_RAMCACHE
5284 load_ramcache();
5285#endif
5286 check_deleted_files();
5287 break ;
5288
5289 case Q_START_SCAN:
5290 check_done = false;
5291 /* fallthrough */
5292 case SYS_TIMEOUT:
5293 if (check_done || !tc_stat.ready)
5294 break ;
5295
5296#ifdef HAVE_TC_RAMCACHE
5297 if (!tc_stat.ramcache && global_settings.tagcache_ram)
5298 {
5299 load_ramcache();
5300 if (global_settings.tagcache_ram == TAGCACHE_RAM_ON)
5301 check_file_refs(global_settings.tagcache_autoupdate);
5302 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
5303 tagcache_build();
5304 }
5305 else
5306#endif /* HAVE_RC_RAMCACHE */
5307 if (global_settings.tagcache_autoupdate)
5308 {
5309 tagcache_build();
5310
5311 /* This will be very slow unless dircache is enabled
5312 or target is flash based, but do it anyway for
5313 consistency. */
5314 check_deleted_files();
5315 }
5316
5317 logf("tagcache check done");
5318
5319 check_done = true;
5320 break ;
5321
5322 case Q_STOP_SCAN:
5323 break ;
5324
5325 case SYS_POWEROFF:
5326 case SYS_REBOOT:
5327 break ;
5328
5329 case SYS_USB_CONNECTED:
5330 logf("USB: TagCache");
5331 usb_acknowledge(SYS_USB_CONNECTED_ACK);
5332 usb_wait_for_disconnect(&tagcache_queue);
5333 break ;
5334 }
5335 }
5336}
5337
5338bool tagcache_prepare_shutdown(void)
5339{
5340 if (tagcache_get_commit_step() > 0)
5341 return false;
5342
5343 tagcache_stop_scan();
5344 while (read_lock || write_lock)
5345 sleep(1);
5346
5347 return true;
5348}
5349
5350void tagcache_shutdown(void)
5351{
5352 /* Flush the command queue. */
5353 run_command_queue(true);
5354
5355#if defined(HAVE_EEPROM_SETTINGS) && defined(HAVE_TC_RAMCACHE)
5356 if (tc_stat.ramcache)
5357 tagcache_dumpsave();
5358#endif
5359}
5360
5361void tagcache_remove_statefile(void)
5362{
5363 remove_db_file(TAGCACHE_STATEFILE);
5364}
5365
5366static int get_progress(void)
5367{
5368 int total_count = -1;
5369
5370#ifdef HAVE_DIRCACHE
5371 struct dircache_info dcinfo;
5372 dircache_get_info(&dcinfo);
5373 if (dcinfo.status != DIRCACHE_IDLE &&
5374 ((!tc_stat.ramcache) || current_tcmh.tch.entry_count == 0))
5375 {
5376 total_count = dcinfo.entry_count;
5377 }
5378 else
5379#endif /* HAVE_DIRCACHE */
5380#ifdef HAVE_TC_RAMCACHE
5381 {
5382 if (tcramcache.hdr && tc_stat.ramcache)
5383 total_count = current_tcmh.tch.entry_count;
5384 }
5385#endif /* HAVE_TC_RAMCACHE */
5386
5387 if (total_count < 0)
5388 return -1;
5389 else if (total_count == 0)
5390 return 0;
5391 else if (processed_dir_count > total_count)
5392 return 100;
5393 else
5394 return processed_dir_count * 100 / total_count;
5395}
5396
5397struct tagcache_stat* tagcache_get_stat(void)
5398{
5399 tc_stat.total_entries = current_tcmh.tch.entry_count;
5400 tc_stat.progress = get_progress();
5401 tc_stat.processed_entries = processed_dir_count;
5402
5403 return &tc_stat;
5404}
5405
5406void tagcache_start_scan(void)
5407{
5408 queue_post(&tagcache_queue, Q_START_SCAN, 0);
5409}
5410
5411bool tagcache_update(void)
5412{
5413 if (!tc_stat.ready)
5414 return false;
5415
5416 queue_post(&tagcache_queue, Q_UPDATE, 0);
5417 return false;
5418}
5419
5420bool tagcache_rebuild(void)
5421{
5422 queue_post(&tagcache_queue, Q_REBUILD, 0);
5423 return false;
5424}
5425
5426void tagcache_stop_scan(void)
5427{
5428 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
5429}
5430
5431#endif /* !__PCTOOL__ */
5432
5433
5434void tagcache_init(void)
5435{
5436 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
5437 memset(¤t_tcmh, 0, sizeof(struct master_header));
5438 filenametag_fd = -1;
5439 write_lock = read_lock = 0;
5440
5441#ifndef __PCTOOL__
5442 strmemccpy(tc_stat.db_path, global_settings.tagcache_db_path,
5443 sizeof(tc_stat.db_path));
5444 mutex_init(&command_queue_mutex);
5445 queue_init(&tagcache_queue, true);
5446 create_thread(tagcache_thread, tagcache_stack,
5447 sizeof(tagcache_stack), 0, tagcache_thread_name
5448 IF_PRIO(, PRIORITY_BACKGROUND)
5449 IF_COP(, CPU));
5450#else
5451 /* use default DB path */
5452 strcpy(tc_stat.db_path, ROCKBOX_DIR);
5453 tc_stat.initialized = true;
5454 allocate_tempbuf();
5455 commit();
5456 free_tempbuf();
5457 tc_stat.ready = check_all_headers();
5458#endif
5459}
5460
5461#ifdef __PCTOOL__
5462void tagcache_reverse_scan(void)
5463{
5464 logf("Checking for deleted files");
5465 check_deleted_files();
5466}
5467#endif
5468
5469bool tagcache_is_initialized(void)
5470{
5471 return tc_stat.initialized;
5472}
5473bool tagcache_is_fully_initialized(void)
5474{
5475 return tc_stat.readyvalid;
5476}
5477bool tagcache_is_usable(void)
5478{
5479 return tc_stat.initialized && tc_stat.ready;
5480}
5481#ifdef HAVE_TC_RAMCACHE
5482bool tagcache_is_in_ram(void)
5483{
5484 return tc_stat.ramcache;
5485}
5486#endif
5487int tagcache_get_commit_step(void)
5488{
5489 return tc_stat.commit_step;
5490}
5491int tagcache_get_max_commit_step(void)
5492{
5493 return (int)(SORTED_TAGS_COUNT)+1;
5494}
5495#endif /*!defined(PLUGIN)*/