#971 workaround to remove and report duplicates as they appear

lib/model/db/db_sqflite.dart

@@ -112,7 +112,7 @@ class SqfliteLocalMediaDb implements LocalMediaDb {
       version: 11,
     );
 
-    final maxIdRows = await _db.rawQuery('SELECT max(id) AS maxId FROM $entryTable');
+    final maxIdRows = await _db.rawQuery('SELECT MAX(id) AS maxId FROM $entryTable');
     _lastId = (maxIdRows.firstOrNull?['maxId'] as int?) ?? 0;
   }
 
@@ -252,27 +252,20 @@ class SqfliteLocalMediaDb implements LocalMediaDb {
     if (entries != null) {
       where += ' AND contentId IN (${entries.map((v) => v.contentId).join(',')})';
     }
-    final rows = await _db.query(
+    final rows = await _db.rawQuery(
-      entryTable,
+      'SELECT *, MAX(id) AS id'
-      where: where,
+      ' FROM $entryTable'
-      whereArgs: [origin, 0],
+      ' WHERE $where'
-      groupBy: 'contentId',
+      ' GROUP BY contentId'
-      having: 'COUNT(id) > 1',
+      ' HAVING COUNT(id) > 1',
+      [origin, 0],
     );
     final duplicates = rows.map(AvesEntry.fromMap).toSet();
-    if (duplicates.isEmpty) {
+    if (duplicates.isNotEmpty) {
-      return {};
+      debugPrint('Found duplicates=$duplicates');
-    }
-
-    debugPrint('Found duplicates=$duplicates');
-    if (entries != null) {
-      // return duplicates among the provided entries
-      final duplicateIds = duplicates.map((v) => v.id).toSet();
-      return entries.where((v) => duplicateIds.contains(v.id)).toSet();
-    } else {
-      // return latest duplicates for each content ID
-      return duplicates.groupFoldBy<int?, AvesEntry>((v) => v.contentId, (prev, v) => prev != null && prev.id > v.id ? prev : v).values.toSet();
     }
+    // return most recent duplicate for each duplicated content ID
+    return duplicates;
   }
 
   // date taken

lib/model/source/media_store_source.dart

@@ -1,5 +1,4 @@
 import 'dart:async';
-import 'dart:math';
 
 import 'package:aves/model/covers.dart';
 import 'package:aves/model/entry/entry.dart';
@@ -159,26 +158,16 @@ class MediaStoreSource extends CollectionSource {
     });
 
     // items to add to the collection
-    final pendingNewEntries = <AvesEntry>{};
+    final newEntries = <AvesEntry>{};
 
     // recover untracked trash items
     debugPrint('$runtimeType refresh ${stopwatch.elapsed} recover untracked entries');
     if (directory == null) {
-      pendingNewEntries.addAll(await recoverUntrackedTrashItems());
+      newEntries.addAll(await recoverUntrackedTrashItems());
     }
 
     // fetch new & modified entries
     debugPrint('$runtimeType refresh ${stopwatch.elapsed} fetch new entries');
-    // refresh after the first 10 entries, then after 100 more, then every 1000 entries
-    var refreshCount = 10;
-    const refreshCountMax = 1000;
-    final allNewEntries = <AvesEntry>{};
-    void addPendingEntries() {
-      allNewEntries.addAll(pendingNewEntries);
-      addEntries(pendingNewEntries);
-      pendingNewEntries.clear();
-    }
-
     mediaStoreService.getEntries(_safeMode, knownDateByContentId, directory: directory).listen(
       (entry) {
         // when discovering modified entry with known content ID,
@@ -187,25 +176,27 @@ class MediaStoreSource extends CollectionSource {
         final existingEntry = knownContentIds.contains(contentId) ? knownLiveEntries.firstWhereOrNull((entry) => entry.contentId == contentId) : null;
         entry.id = existingEntry?.id ?? localMediaDb.nextId;
 
-        pendingNewEntries.add(entry);
+        newEntries.add(entry);
-        if (pendingNewEntries.length >= refreshCount) {
-          refreshCount = min(refreshCount * 10, refreshCountMax);
-          addPendingEntries();
-        }
       },
       onDone: () async {
-        addPendingEntries();
+        if (newEntries.isNotEmpty) {
-
-        if (allNewEntries.isNotEmpty) {
           debugPrint('$runtimeType refresh ${stopwatch.elapsed} save new entries');
-          await localMediaDb.insertEntries(allNewEntries);
+          await localMediaDb.insertEntries(newEntries);
 
-          // TODO TLAD [971] check duplicates
+          // TODO TLAD find duplication cause
-          final duplicates = await localMediaDb.searchLiveDuplicates(EntryOrigins.mediaStoreContent, allNewEntries);
+          final duplicates = await localMediaDb.searchLiveDuplicates(EntryOrigins.mediaStoreContent, newEntries);
           if (duplicates.isNotEmpty) {
             unawaited(reportService.recordError(Exception('Loading entries yielded duplicates=${duplicates.join(', ')}'), StackTrace.current));
+            // post-error cleanup
+            await localMediaDb.removeIds(duplicates.map((v) => v.id).toSet());
+            for (final duplicate in duplicates) {
+              final duplicateId = duplicate.id;
+              newEntries.removeWhere((v) => duplicateId == v.id);
+            }
           }
 
+          addEntries(newEntries);
+
           // new entries include existing entries with obsolete paths
           // so directories may be added, but also removed or simply have their content summary changed
           invalidateAlbumFilterSummary();
@@ -230,7 +221,7 @@ class MediaStoreSource extends CollectionSource {
         notifyAlbumsChanged();
 
         debugPrint('$runtimeType refresh ${stopwatch.elapsed} done');
-        unawaited(reportService.log('Source refresh complete in ${stopwatch.elapsed.inSeconds}s for ${knownEntries.length} known, ${allNewEntries.length} new, ${removedEntries.length} removed'));
+        unawaited(reportService.log('Source refresh complete in ${stopwatch.elapsed.inSeconds}s for ${knownEntries.length} known, ${newEntries.length} new, ${removedEntries.length} removed'));
       },
       onError: (error) => debugPrint('$runtimeType stream error=$error'),
     );
@@ -248,7 +239,7 @@ class MediaStoreSource extends CollectionSource {
     state = SourceState.loading;
 
     debugPrint('$runtimeType refreshUris ${changedUris.length} uris');
-    final uriByContentId = Map.fromEntries(changedUris.map((uri) {
+    final changedUriByContentId = Map.fromEntries(changedUris.map((uri) {
       final pathSegments = Uri.parse(uri).pathSegments;
       // e.g. URI `content://media/` has no path segment
       if (pathSegments.isEmpty) return null;
@@ -259,16 +250,16 @@ class MediaStoreSource extends CollectionSource {
     }).whereNotNull());
 
     // clean up obsolete entries
-    final obsoleteContentIds = (await mediaStoreService.checkObsoleteContentIds(uriByContentId.keys.toList())).toSet();
+    final obsoleteContentIds = (await mediaStoreService.checkObsoleteContentIds(changedUriByContentId.keys.toList())).toSet();
-    final obsoleteUris = obsoleteContentIds.map((contentId) => uriByContentId[contentId]).whereNotNull().toSet();
+    final obsoleteUris = obsoleteContentIds.map((contentId) => changedUriByContentId[contentId]).whereNotNull().toSet();
     await removeEntries(obsoleteUris, includeTrash: false);
-    obsoleteContentIds.forEach(uriByContentId.remove);
+    obsoleteContentIds.forEach(changedUriByContentId.remove);
 
     // fetch new entries
     final tempUris = <String>{};
     final newEntries = <AvesEntry>{}, entriesToRefresh = <AvesEntry>{};
     final existingDirectories = <String>{};
-    for (final kv in uriByContentId.entries) {
+    for (final kv in changedUriByContentId.entries) {
       final contentId = kv.key;
       final uri = kv.value;
       final sourceEntry = await mediaFetchService.getEntry(uri, null);
@@ -309,15 +300,22 @@ class MediaStoreSource extends CollectionSource {
     state = SourceState.ready;
 
     if (newEntries.isNotEmpty) {
-      addEntries(newEntries);
       await localMediaDb.insertEntries(newEntries);
 
-      // TODO TLAD [971] check duplicates
+      // TODO TLAD find duplication cause
       final duplicates = await localMediaDb.searchLiveDuplicates(EntryOrigins.mediaStoreContent, newEntries);
       if (duplicates.isNotEmpty) {
         unawaited(reportService.recordError(Exception('Refreshing entries yielded duplicates=${duplicates.join(', ')}'), StackTrace.current));
+        // post-error cleanup
+        await localMediaDb.removeIds(duplicates.map((v) => v.id).toSet());
+        for (final duplicate in duplicates) {
+          final duplicateId = duplicate.id;
+          newEntries.removeWhere((v) => duplicateId == v.id);
+          tempUris.add(duplicate.uri);
+        }
       }
 
+      addEntries(newEntries);
       await analyze(analysisController, entries: newEntries);
     }
 

lib/widgets/collection/collection_grid.dart

@@ -20,6 +20,7 @@ import 'package:aves/widgets/collection/draggable_thumb_label.dart';
 import 'package:aves/widgets/collection/grid/list_details_theme.dart';
 import 'package:aves/widgets/collection/grid/section_layout.dart';
 import 'package:aves/widgets/collection/grid/tile.dart';
+import 'package:aves/widgets/common/action_mixins/feedback.dart';
 import 'package:aves/widgets/common/basic/draggable_scrollbar/scrollbar.dart';
 import 'package:aves/widgets/common/basic/insets.dart';
 import 'package:aves/widgets/common/behaviour/routes.dart';
@@ -587,7 +588,13 @@ class _CollectionScrollViewState extends State<_CollectionScrollView> with Widge
       valueListenable: collection.source.stateNotifier,
       builder: (context, sourceState, child) {
         if (sourceState == SourceState.loading) {
-          return const SizedBox();
+          return EmptyContent(
+            text: context.l10n.sourceStateLoading,
+            bottom: const Padding(
+              padding: EdgeInsets.only(top: 16),
+              child: ReportProgressIndicator(),
+            ),
+          );
         }
 
         return FutureBuilder<bool>(

lib/widgets/common/action_mixins/feedback.dart

@@ -166,6 +166,9 @@ class ReportOverlay<T> extends StatefulWidget {
   final VoidCallback? onCancel;
   final void Function(Set<T> processed) onDone;
 
+  static const double diameter = 160.0;
+  static const double strokeWidth = 8.0;
+
   const ReportOverlay({
     super.key,
     required this.opStream,
@@ -186,8 +189,6 @@ class _ReportOverlayState<T> extends State<ReportOverlay<T>> with SingleTickerPr
   Stream<T> get opStream => widget.opStream;
 
   static const double fontSize = 18.0;
-  static const double diameter = 160.0;
-  static const double strokeWidth = 8.0;
 
   @override
   void initState() {
@@ -222,6 +223,8 @@ class _ReportOverlayState<T> extends State<ReportOverlay<T>> with SingleTickerPr
 
   @override
   Widget build(BuildContext context) {
+    const diameter = ReportOverlay.diameter;
+    const strokeWidth = ReportOverlay.strokeWidth;
     final percentFormatter = NumberFormat.percentPattern(context.locale);
 
     final theme = Theme.of(context);
@@ -249,16 +252,7 @@ class _ReportOverlayState<T> extends State<ReportOverlay<T>> with SingleTickerPr
                     shape: BoxShape.circle,
                   ),
                 ),
-                if (animate)
+                if (animate) const ReportProgressIndicator(opacity: .1),
-                  Container(
-                    width: diameter,
-                    height: diameter,
-                    padding: const EdgeInsets.all(strokeWidth / 2),
-                    child: CircularProgressIndicator(
-                      color: progressColor.withOpacity(.1),
-                      strokeWidth: strokeWidth,
-                    ),
-                  ),
                 CircularPercentIndicator(
                   percent: percent,
                   lineWidth: strokeWidth,
@@ -301,6 +295,32 @@ class _ReportOverlayState<T> extends State<ReportOverlay<T>> with SingleTickerPr
   }
 }
 
+class ReportProgressIndicator extends StatelessWidget {
+  final double opacity;
+
+  const ReportProgressIndicator({
+    super.key,
+    this.opacity = 1,
+  });
+
+  @override
+  Widget build(BuildContext context) {
+    const diameter = ReportOverlay.diameter;
+    const strokeWidth = ReportOverlay.strokeWidth;
+    final progressColor = Theme.of(context).colorScheme.primary;
+
+    return Container(
+      width: diameter,
+      height: diameter,
+      padding: const EdgeInsets.all(strokeWidth / 2),
+      child: CircularProgressIndicator(
+        color: progressColor.withOpacity(opacity),
+        strokeWidth: strokeWidth,
+      ),
+    );
+  }
+}
+
 class _FeedbackMessage extends StatefulWidget {
   final FeedbackType type;
   final String message;