From 2ba264ec098e6479a81c9b7d22757654185ec173 Mon Sep 17 00:00:00 2001 From: QubaB Date: Sat, 27 Sep 2025 09:58:33 +0200 Subject: [PATCH 01/35] started work on new assetCacheAll - cache which holds assets all the time and removes all duplicities Now implemented only adding to cache during read of note fromJson. Calculated is previewHash from first 100kB of files for fast recognition of assets which are duplicated (all assets which are saved after note is reedited) --- lib/components/canvas/_asset_cache.dart | 323 +++++++++++++++++- lib/components/canvas/image/editor_image.dart | 6 + .../canvas/image/pdf_editor_image.dart | 4 + .../canvas/image/png_editor_image.dart | 14 +- .../canvas/image/svg_editor_image.dart | 4 + lib/data/editor/editor_core_info.dart | 20 +- lib/data/editor/page.dart | 7 + lib/pages/editor/editor.dart | 27 +- 8 files changed, 388 insertions(+), 17 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index eec7519d24..b68100f990 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -1,6 +1,6 @@ +import 'dart:async'; import 'dart:convert'; import 'dart:io'; - import 'package:collection/collection.dart'; import 'package:flutter/painting.dart'; import 'package:logging/logging.dart'; @@ -134,3 +134,324 @@ class OrderedAssetCache { } } } +/////////////////////////////////////////////////////////////////////////// +/// New approach to cache + +// class returning preview data +class PreviewResult { + final int previewHash; + final int fileSize; + + PreviewResult(this.previewHash, this.fileSize); +} + + +// combine preview hash and size to "preliminary hash" +int makeCompositeKey(int previewHash, int fileSize) { + // Shift size into high bits, preview hash stays in low bits + // -> reduces collisions compared to just XOR + return (fileSize.hashCode << 32) ^ previewHash; +} + +// object in cache +class CacheItem { + final Object value; + int? previewHash; // quick hash (from first 100KB bytes) + int? hash; // hash can be calculated later + + // for files only + final int? fileSize; + final String? filePath; // only for files - for fast comparison without reading file contents + + int _refCount = 0; // number of references + bool _released = false; + + CacheItem(this.value, + {this.hash, this.filePath, this.previewHash, this.fileSize}); + + + // increase use of item + void addUse() { + if (_released) throw StateError('Trying to add use of released CacheItem'); + _refCount++; + } + + // when asset is released (no more used) + void freeUse() { + if (_refCount > 0) _refCount--; + if (_refCount == 0) _released = true; + } + + bool get isUnused => _refCount == 0; + bool get isReleased => _released; + + @override + bool operator ==(Object other) { + if (other is! CacheItem) return false; + + // compare hashes it is precise + if (hash != null && other.hash != null) { + return hash == other.hash; + } + + // Quick check using previewHash + if (previewHash != null && other.previewHash != null) { + if (previewHash != other.previewHash) { + // preview hashes do not match - assets are different + return false; + } + } + if (filePath != null && other.filePath != null) { + if (filePath == other.filePath) { + // both file paths are the same + return true; + } + } + return false; // consider not equal + } + + + + +// @override +// int? get hash => filePath?.hash ?? hash; + + @override + String toString() => + 'CacheItem(path: $filePath, preview=$previewHash, full=$hash, refs=$_refCount, released=$_released)'; +} + +// cache manager +class AssetCacheAll { + final List _items = []; + final Map _aliasMap = {}; // duplicit indices point to first indice - updated in finalize + final Map _previewHashIndex = {}; // Map from previewHash → first index in _items + + + final log = Logger('OrderedAssetCache'); + + // calculate hash of bytes (all) + int calculateHash(List bytes) { // fnv1a + int hash = 0x811C9DC5; + for (var b in bytes) { + hash ^= b; + hash = (hash * 0x01000193) & 0xFFFFFFFF; + } + return hash; + } + +// Compute a quick hash based on the first 100 KB of the file. +// This can be done synchronously to quickly filter duplicates. +// calculate preview hash of file + PreviewResult getFilePreviewHash(File file) { + final stat = file.statSync(); // get file metadata + final fileSize = stat.size; + + final raf = file.openSync(mode: FileMode.read); + try { + // read either the whole file if small, or just the first 100 KB + final toRead = fileSize < 100 * 1024 ? fileSize : 100 * 1024; + final bytes = raf.readSync(toRead); + final previewHash = calculateHash(bytes); + return PreviewResult((fileSize.hashCode << 32) ^ previewHash, // hash + fileSize); // file size + } finally { + raf.closeSync(); + } + } + + // add to cache but read only small part of files - used when reading note from disk + // full hashes are established later + int addSync(Object value) { + if (value is File) { + final path = value.path; + final previewResult=getFilePreviewHash(value); // calculate preliminary hash of file + + // Check if already cached + if (_previewHashIndex.containsKey(previewResult.previewHash)) { + final existingIndex = _previewHashIndex[previewResult.previewHash]!; + _items[existingIndex].addUse(); + return existingIndex; + } + + final existingPathIndex = _items.indexWhere((i) => i.filePath == path); + if (existingPathIndex != -1) return existingPathIndex; + + final newItem = CacheItem(value, + filePath: value.path, + previewHash: previewResult.previewHash, + fileSize: previewResult.fileSize)..addUse(); + _items.add(newItem); + final index = _items.length - 1; + _previewHashIndex[previewResult.previewHash] = index; // add to previously hashed + return index; + + } else if (value is FileImage) { + final path = value.file.path; + final File file = File(path); + final previewResult=getFilePreviewHash(file); // calculate preliminary hash of file + + // Check if already cached + if (_previewHashIndex.containsKey(previewResult.previewHash)) { + final existingIndex = _previewHashIndex[previewResult.previewHash]!; + _items[existingIndex].addUse(); + return existingIndex; + } + + final existingPathIndex = _items.indexWhere((i) => i.filePath == path); + if (existingPathIndex != -1) return existingPathIndex; + + final newItem = CacheItem(value, + filePath: path, + previewHash: previewResult.previewHash, + fileSize: previewResult.fileSize)..addUse(); + _items.add(newItem); + final index = _items.length - 1; + _previewHashIndex[previewResult.previewHash] = index; // add to previously hashed + return index; + } else if (value is MemoryImage) { // file images are first compared by file path + final hash = calculateHash(value.bytes); + final newItem = CacheItem(value, hash: hash)..addUse(); + + final existingHashIndex = _items.indexOf(newItem); + if (existingHashIndex != -1) return existingHashIndex; + + _items.add(newItem); + final index = _items.length - 1; + return index; + } else if (value is String){ + // directly calculate hash + final newItem = CacheItem(value, hash: value.hashCode)..addUse(); + final existingHashIndex = _items.indexOf(newItem); + if (existingHashIndex != -1) return existingHashIndex; + _items.add(newItem); + final index = _items.length - 1; + return index; + } else { + throw Exception( + 'OrderedAssetCache.getBytes: unknown type ${value.runtimeType}'); + } + } + + + Future add(Object value) async { + if (value is File) { // files are first compared by file path + final path = value.path; + + // 1. Fast path check + final existingPathIndex = + _items.indexWhere((i) => i.filePath == path); + if (existingPathIndex != -1) return existingPathIndex; + + // 2. Otherwise compute expensive content hash + final bytes = await value.readAsBytes(); + final hash = calculateHash(bytes); + + final newItem = CacheItem(value, hash: hash, filePath: path); + + final existingHashIndex = _items.indexOf(newItem); + if (existingHashIndex != -1) return existingHashIndex; + + _items.add(newItem); + return _items.length - 1; + } else if (value is FileImage) { // file images are first compared by file path + final path = value.file.path; + + // 1. Fast path check + final existingPathIndex = + _items.indexWhere((i) => i.filePath == path); + if (existingPathIndex != -1) return existingPathIndex; + + // 2. Otherwise compute expensive content hash + final bytes = await value.file.readAsBytes(); + final hash = calculateHash(bytes); + + final newItem = CacheItem(value, hash: hash, filePath: path); + + final existingHashIndex = _items.indexOf(newItem); + if (existingHashIndex != -1) return existingHashIndex; + + _items.add(newItem); + return _items.length - 1; + } else if (value is MemoryImage) { // file images are first compared by file path + final hash = calculateHash(value.bytes); + + final newItem = CacheItem(value, hash: hash); + + final existingHashIndex = _items.indexOf(newItem); + if (existingHashIndex != -1) return existingHashIndex; + + _items.add(newItem); + return _items.length - 1; + } else { + final hash = value.hashCode; // string + final newItem = CacheItem(value, hash: hash); + + final existingIndex = _items.indexOf(newItem); + if (existingIndex != -1) return existingIndex; + + _items.add(newItem); + return _items.length - 1; + } + } + + /// The number of (distinct) items in the cache. + int get length => _items.length; + bool get isEmpty => _items.isEmpty; + bool get isNotEmpty => _items.isNotEmpty; + + /// Converts the item at position [indexIn] + /// to bytes and returns them. + Future> getBytes(int indexIn) async { + final index = resolveIndex(indexIn); // find first occurence in cache to avoid duplicities + final item = _items[index].value; + if (item is List) { + return item; + } else if (item is File) { + return item.readAsBytes(); + } else if (item is String) { + return utf8.encode(item); + } else if (item is MemoryImage) { + return item.bytes; + } else if (item is FileImage) { + return item.file.readAsBytes(); + } else { + throw Exception( + 'OrderedAssetCache.getBytes: unknown type ${item.runtimeType}'); + } + } + + // finalize cache after it was filled using addSync - without calculation of hashes + Future finalize() async { + final Map seenHashes = {}; // hash points to first index + + for (int i = 0; i < _items.length; i++) { + final item = _items[i]; + int hash; + int? hashItem = item.hash; + if (hashItem == 0) { + final bytes = await getBytes(i); + hash = calculateHash(bytes); + _items[i] = CacheItem(item.value, hash: hash, filePath: item.filePath, fileSize: item.fileSize); + } + else { + hash=hashItem!; + } + + if (seenHashes.containsKey(hash)) { + // už existuje → aliasuj na první výskyt + _aliasMap[i] = seenHashes[hash]!; + } else { + seenHashes[hash] = i; + } + } + } + + /// Vrátí reálný index (přes alias mapu) + int resolveIndex(int index) { + return _aliasMap[index] ?? index; + } + + @override + String toString() => _items.toString(); +} diff --git a/lib/components/canvas/image/editor_image.dart b/lib/components/canvas/image/editor_image.dart index 1929724b78..a7afd68494 100644 --- a/lib/components/canvas/image/editor_image.dart +++ b/lib/components/canvas/image/editor_image.dart @@ -34,6 +34,7 @@ sealed class EditorImage extends ChangeNotifier { final String extension; final AssetCache assetCache; + final AssetCacheAll assetCacheAll; bool _isThumbnail = false; bool get isThumbnail => _isThumbnail; @@ -89,6 +90,7 @@ sealed class EditorImage extends ChangeNotifier { EditorImage({ required this.id, required this.assetCache, + required this.assetCacheAll, required this.extension, required this.pageIndex, required this.pageSize, @@ -113,6 +115,7 @@ sealed class EditorImage extends ChangeNotifier { bool isThumbnail = false, required String sbnPath, required AssetCache assetCache, + required AssetCacheAll assetCacheAll, }) { String? extension = json['e']; if (extension == '.svg') { @@ -122,6 +125,7 @@ sealed class EditorImage extends ChangeNotifier { isThumbnail: isThumbnail, sbnPath: sbnPath, assetCache: assetCache, + assetCacheAll: assetCacheAll, ); } else if (extension == '.pdf') { return PdfEditorImage.fromJson( @@ -130,6 +134,7 @@ sealed class EditorImage extends ChangeNotifier { isThumbnail: isThumbnail, sbnPath: sbnPath, assetCache: assetCache, + assetCacheAll: assetCacheAll, ); } else { return PngEditorImage.fromJson( @@ -138,6 +143,7 @@ sealed class EditorImage extends ChangeNotifier { isThumbnail: isThumbnail, sbnPath: sbnPath, assetCache: assetCache, + assetCacheAll: assetCacheAll, ); } } diff --git a/lib/components/canvas/image/pdf_editor_image.dart b/lib/components/canvas/image/pdf_editor_image.dart index 8b0bf3b5a1..5a1bd196ea 100644 --- a/lib/components/canvas/image/pdf_editor_image.dart +++ b/lib/components/canvas/image/pdf_editor_image.dart @@ -15,6 +15,7 @@ class PdfEditorImage extends EditorImage { PdfEditorImage({ required super.id, required super.assetCache, + required super.assetCacheAll, required this.pdfBytes, required this.pdfFile, required this.pdfPage, @@ -45,6 +46,7 @@ class PdfEditorImage extends EditorImage { bool isThumbnail = false, required String sbnPath, required AssetCache assetCache, + required AssetCacheAll assetCacheAll, }) { String? extension = json['e'] as String?; assert(extension == null || extension == '.pdf'); @@ -71,6 +73,7 @@ class PdfEditorImage extends EditorImage { id: json['id'] ?? -1, // -1 will be replaced by EditorCoreInfo._handleEmptyImageIds() assetCache: assetCache, + assetCacheAll: assetCacheAll, pdfBytes: pdfBytes, pdfFile: pdfFile, pdfPage: json['pdfi'], @@ -184,6 +187,7 @@ class PdfEditorImage extends EditorImage { PdfEditorImage copy() => PdfEditorImage( id: id, assetCache: assetCache, + assetCacheAll: assetCacheAll, pdfBytes: pdfBytes, pdfPage: pdfPage, pdfFile: pdfFile, diff --git a/lib/components/canvas/image/png_editor_image.dart b/lib/components/canvas/image/png_editor_image.dart index bec7a4d307..c27e51a770 100644 --- a/lib/components/canvas/image/png_editor_image.dart +++ b/lib/components/canvas/image/png_editor_image.dart @@ -24,6 +24,7 @@ class PngEditorImage extends EditorImage { PngEditorImage({ required super.id, required super.assetCache, + required super.assetCacheAll, required super.extension, required this.imageProvider, required super.pageIndex, @@ -49,17 +50,20 @@ class PngEditorImage extends EditorImage { bool isThumbnail = false, required String sbnPath, required AssetCache assetCache, + required AssetCacheAll assetCacheAll, }) { - final assetIndex = json['a'] as int?; + final assetIndexJson = json['a'] as int?; final Uint8List? bytes; + final int? assetIndex; File? imageFile; - if (assetIndex != null) { + if (assetIndexJson != null) { if (inlineAssets == null) { imageFile = - FileManager.getFile('$sbnPath${Editor.extension}.$assetIndex'); + FileManager.getFile('$sbnPath${Editor.extension}.$assetIndexJson'); + assetIndex=assetCacheAll.addSync(imageFile); bytes = assetCache.get(imageFile); } else { - bytes = inlineAssets[assetIndex]; + bytes = inlineAssets[assetIndexJson]; } } else if (json['b'] != null) { bytes = Uint8List.fromList((json['b'] as List).cast()); @@ -76,6 +80,7 @@ class PngEditorImage extends EditorImage { // -1 will be replaced by [EditorCoreInfo._handleEmptyImageIds()] id: json['id'] ?? -1, assetCache: assetCache, + assetCacheAll: assetCacheAll, extension: json['e'] ?? '.jpg', imageProvider: bytes != null ? MemoryImage(bytes) as ImageProvider @@ -219,6 +224,7 @@ class PngEditorImage extends EditorImage { PngEditorImage copy() => PngEditorImage( id: id, assetCache: assetCache, + assetCacheAll: assetCacheAll, extension: extension, imageProvider: imageProvider, pageIndex: pageIndex, diff --git a/lib/components/canvas/image/svg_editor_image.dart b/lib/components/canvas/image/svg_editor_image.dart index 5303b0a9b8..b74417d8c0 100644 --- a/lib/components/canvas/image/svg_editor_image.dart +++ b/lib/components/canvas/image/svg_editor_image.dart @@ -12,6 +12,7 @@ class SvgEditorImage extends EditorImage { SvgEditorImage({ required super.id, required super.assetCache, + required super.assetCacheAll, required String? svgString, required File? svgFile, required super.pageIndex, @@ -45,6 +46,7 @@ class SvgEditorImage extends EditorImage { bool isThumbnail = false, required String sbnPath, required AssetCache assetCache, + required AssetCacheAll assetCacheAll, }) { String? extension = json['e'] as String?; assert(extension == null || extension == '.svg'); @@ -71,6 +73,7 @@ class SvgEditorImage extends EditorImage { id: json['id'] ?? -1, // -1 will be replaced by EditorCoreInfo._handleEmptyImageIds() assetCache: assetCache, + assetCacheAll: assetCacheAll, svgString: svgString, svgFile: svgFile, pageIndex: json['i'] ?? 0, @@ -195,6 +198,7 @@ class SvgEditorImage extends EditorImage { id: id, // ignore: deprecated_member_use_from_same_package assetCache: assetCache, + assetCacheAll: assetCacheAll, svgString: svgData.string, svgFile: svgData.file, pageIndex: pageIndex, diff --git a/lib/data/editor/editor_core_info.dart b/lib/data/editor/editor_core_info.dart index b12d215a7e..8ae5d01ff7 100644 --- a/lib/data/editor/editor_core_info.dart +++ b/lib/data/editor/editor_core_info.dart @@ -58,6 +58,7 @@ class EditorCoreInfo { String get fileName => filePath.substring(filePath.lastIndexOf('/') + 1); AssetCache assetCache; + AssetCacheAll assetCacheAll; int nextImageId; Color? backgroundColor; CanvasBackgroundPattern backgroundPattern; @@ -80,6 +81,7 @@ class EditorCoreInfo { pages: [], initialPageIndex: null, assetCache: null, + assetCacheAll: null, ).._migrateOldStrokesAndImages( fileVersion: sbnVersion, strokesJson: null, @@ -100,7 +102,8 @@ class EditorCoreInfo { lineHeight = stows.lastLineHeight.value, lineThickness = stows.lastLineThickness.value, pages = [], - assetCache = AssetCache(); + assetCache = AssetCache(), + assetCacheAll = AssetCacheAll(); EditorCoreInfo._({ required this.filePath, @@ -114,7 +117,10 @@ class EditorCoreInfo { required this.pages, required this.initialPageIndex, required AssetCache? assetCache, - }) : assetCache = assetCache ?? AssetCache() { + required AssetCacheAll? assetCacheAll, + }) : assetCache = assetCache ?? AssetCache(), + assetCacheAll = assetCacheAll ?? AssetCacheAll() + { _handleEmptyImageIds(); } @@ -157,6 +163,7 @@ class EditorCoreInfo { } final assetCache = AssetCache(); + final assetCacheAll = AssetCacheAll(); return EditorCoreInfo._( filePath: filePath, @@ -181,9 +188,11 @@ class EditorCoreInfo { fileVersion: fileVersion, sbnPath: filePath, assetCache: assetCache, + assetCacheAll: assetCacheAll, ), initialPageIndex: json['c'] as int?, assetCache: assetCache, + assetCacheAll: assetCacheAll, ) .._migrateOldStrokesAndImages( fileVersion: fileVersion, @@ -209,7 +218,8 @@ class EditorCoreInfo { lineHeight = stows.lastLineHeight.value, lineThickness = stows.lastLineThickness.value, pages = [], - assetCache = AssetCache() { + assetCache = AssetCache(), + assetCacheAll = AssetCacheAll(){ _migrateOldStrokesAndImages( fileVersion: 0, strokesJson: json, @@ -228,6 +238,7 @@ class EditorCoreInfo { required int fileVersion, required String sbnPath, required AssetCache assetCache, + required AssetCacheAll assetCacheAll, }) { if (pages == null || pages.isEmpty) return []; if (pages[0] is List) { @@ -249,6 +260,7 @@ class EditorCoreInfo { fileVersion: fileVersion, sbnPath: sbnPath, assetCache: assetCache, + assetCacheAll: assetCacheAll, )) .toList(); } @@ -306,6 +318,7 @@ class EditorCoreInfo { onlyFirstPage: onlyFirstPage, sbnPath: filePath, assetCache: assetCache, + assetCacheAll: assetCacheAll, ); for (EditorImage image in images) { if (onlyFirstPage) assert(image.pageIndex == 0); @@ -561,6 +574,7 @@ class EditorCoreInfo { pages: pages ?? this.pages, initialPageIndex: initialPageIndex, assetCache: assetCache, + assetCacheAll: assetCacheAll, ); } } diff --git a/lib/data/editor/page.dart b/lib/data/editor/page.dart index d569e87f04..b35d9ee6fa 100644 --- a/lib/data/editor/page.dart +++ b/lib/data/editor/page.dart @@ -131,6 +131,7 @@ class EditorPage extends ChangeNotifier implements HasSize { required int fileVersion, required String sbnPath, required AssetCache assetCache, + required AssetCacheAll assetCacheAll, }) { final size = Size(json['w'] ?? defaultWidth, json['h'] ?? defaultHeight); return EditorPage( @@ -148,6 +149,7 @@ class EditorPage extends ChangeNotifier implements HasSize { onlyFirstPage: false, sbnPath: sbnPath, assetCache: assetCache, + assetCacheAll: assetCacheAll, ), quill: QuillStruct( controller: json['q'] != null @@ -165,6 +167,7 @@ class EditorPage extends ChangeNotifier implements HasSize { isThumbnail: false, sbnPath: sbnPath, assetCache: assetCache, + assetCacheAll: assetCacheAll, ) : null, ); @@ -243,6 +246,7 @@ class EditorPage extends ChangeNotifier implements HasSize { required bool onlyFirstPage, required String sbnPath, required AssetCache assetCache, + required AssetCacheAll assetCacheAll, }) => images ?.cast>() @@ -254,6 +258,7 @@ class EditorPage extends ChangeNotifier implements HasSize { isThumbnail: isThumbnail, sbnPath: sbnPath, assetCache: assetCache, + assetCacheAll: assetCacheAll, ); }) .where((element) => element != null) @@ -267,6 +272,7 @@ class EditorPage extends ChangeNotifier implements HasSize { required bool isThumbnail, required String sbnPath, required AssetCache assetCache, + required AssetCacheAll assetCacheAll, }) => EditorImage.fromJson( json, @@ -274,6 +280,7 @@ class EditorPage extends ChangeNotifier implements HasSize { isThumbnail: isThumbnail, sbnPath: sbnPath, assetCache: assetCache, + assetCacheAll: assetCacheAll, ); /// Triggers a redraw of the strokes. If you need to redraw images, diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index e35d1cbb62..c04da993dc 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -1082,10 +1082,12 @@ class EditorState extends State { // use the Select tool so that the user can move the new image currentTool = Select.currentSelect; - List images = [ - for (final _PhotoInfo photoInfo in photoInfos) - if (photoInfo.extension == '.svg') - SvgEditorImage( + final List images = []; + for (final _PhotoInfo photoInfo in photoInfos) { + if (photoInfo.extension == '.svg') { + // add photo to cache + int cacheId = await coreInfo.assetCacheAll.add(photoInfo.bytes); + images.add(SvgEditorImage( id: coreInfo.nextImageId++, svgString: utf8.decode(photoInfo.bytes), svgFile: null, @@ -1096,9 +1098,13 @@ class EditorState extends State { onMiscChange: autosaveAfterDelay, onLoad: () => setState(() {}), assetCache: coreInfo.assetCache, - ) - else - PngEditorImage( + assetCacheAll: coreInfo.assetCacheAll, + )); + } + else { + final mImage=MemoryImage(photoInfo.bytes); + int cacheId = await coreInfo.assetCacheAll.add(mImage); + images.add(PngEditorImage( id: coreInfo.nextImageId++, extension: photoInfo.extension, imageProvider: MemoryImage(photoInfo.bytes), @@ -1109,8 +1115,10 @@ class EditorState extends State { onMiscChange: autosaveAfterDelay, onLoad: () => setState(() {}), assetCache: coreInfo.assetCache, - ), - ]; + assetCacheAll: coreInfo.assetCacheAll, + )); + } + } history.recordChange(EditorHistoryItem( type: EditorHistoryItemType.draw, @@ -1225,6 +1233,7 @@ class EditorState extends State { onMiscChange: autosaveAfterDelay, onLoad: () => setState(() {}), assetCache: coreInfo.assetCache, + assetCacheAll: coreInfo.assetCacheAll, ); coreInfo.pages.add(page); history.recordChange(EditorHistoryItem( From ce14a043eed211b06a18a69c87b57e3c753d87dc Mon Sep 17 00:00:00 2001 From: QubaB Date: Sat, 27 Sep 2025 17:20:48 +0200 Subject: [PATCH 02/35] implemented pdfDocument in asset cache so it is accessible from all pages as single object started implementation of lazy ImageProvider --- lib/components/canvas/_asset_cache.dart | 109 ++++++++++++++++-- .../canvas/image/pdf_editor_image.dart | 36 ++++-- .../canvas/image/png_editor_image.dart | 26 ++++- lib/pages/editor/editor.dart | 4 + test/isolate_message_test.dart | 5 + test/tools_select_test.dart | 3 + 6 files changed, 165 insertions(+), 18 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index b68100f990..7ac75cb247 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -1,9 +1,11 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'dart:typed_data'; import 'package:collection/collection.dart'; import 'package:flutter/painting.dart'; import 'package:logging/logging.dart'; +import 'package:pdfrx/pdfrx.dart'; import 'package:saber/components/canvas/image/editor_image.dart'; /// A cache for assets that are loaded from disk. @@ -158,6 +160,8 @@ class CacheItem { final Object value; int? previewHash; // quick hash (from first 100KB bytes) int? hash; // hash can be calculated later + ImageProvider? _imageProvider; // image provider for png, svg + PdfDocument? _pdfDocument; // pdf document provider for pdf // for files only final int? fileSize; @@ -227,9 +231,86 @@ class AssetCacheAll { final Map _aliasMap = {}; // duplicit indices point to first indice - updated in finalize final Map _previewHashIndex = {}; // Map from previewHash → first index in _items + final Map> _pendingProviders = {}; // holds currently opening futures to avoid duplicates + final Map> _openingDocs = {}; // Holds currently opening futures to avoid duplicate opens + + final log = Logger('OrderedAssetCache'); + // return pdfDocument of asset it is lazy because it take some time to do it + Future getPdfDocument(int assetId) { + // if already opened, return it immediately + final item = _items[assetId]; + if (item._pdfDocument != null) return Future.value(item._pdfDocument!); + + // if someone else is already opening this doc, return their future + final pending = _openingDocs[assetId]; + if (pending != null) return pending; + + // otherwise start opening + final future = _openPdfDocument(item); + _openingDocs[assetId] = future; + + // when done, store the PdfDocument in the CacheItem and remove from _openingDocs + future.then((doc) { + item._pdfDocument = doc; + _openingDocs.remove(assetId); + }); + + return future; + } + + // open pdf document + Future _openPdfDocument(CacheItem item) async { + if (item.filePath != null) { + return PdfDocument.openFile(item.filePath!); + } else if (item.value is Uint8List) { + return PdfDocument.openData(item.value as Uint8List); + } else { + throw StateError('Asset is not a PDF'); + } + } + + Future getImageProvider(int assetId) { + // return cached provider if already available + final item = _items[assetId]; + if (item._imageProvider != null) return Future.value(item._imageProvider!); + + // if someone else is already creating it, return the pending future + final pending = _pendingProviders[assetId]; + if (pending != null) return pending; + + // otherwise, create the future + final future = _createImageProvider(item); + _pendingProviders[assetId] = future; + + // when done, store the provider and remove from pending + future.then((provider) { + item._imageProvider = provider; + _pendingProviders.remove(assetId); + }); + + return future; + } + + Future _createImageProvider(CacheItem item) async { + if (item.value is File) { + return FileImage(item.value as File); + } else if (item.value is Uint8List) { + return MemoryImage(item.value as Uint8List); + } else if (item.value is MemoryImage) { + return item.value as MemoryImage; + } else if (item.value is FileImage) { + return item.value as FileImage; + } else { + throw UnsupportedError("Unsupported type for ImageProvider: ${item.value.runtimeType}"); + } + } + + + + // calculate hash of bytes (all) int calculateHash(List bytes) { // fnv1a int hash = 0x811C9DC5; @@ -264,9 +345,17 @@ class AssetCacheAll { // full hashes are established later int addSync(Object value) { if (value is File) { + log.info('allCache.addSync: value = $value'); final path = value.path; final previewResult=getFilePreviewHash(value); // calculate preliminary hash of file + final existingPathIndex = _items.indexWhere((i) => i.filePath == path); + if (existingPathIndex != -1) { + _items[existingPathIndex].addUse(); + log.info('allCache.addSync: already in cache {$_items[existingPathIndex]._refCount}'); + return existingPathIndex; + } + // Check if already cached if (_previewHashIndex.containsKey(previewResult.previewHash)) { final existingIndex = _previewHashIndex[previewResult.previewHash]!; @@ -274,18 +363,14 @@ class AssetCacheAll { return existingIndex; } - final existingPathIndex = _items.indexWhere((i) => i.filePath == path); - if (existingPathIndex != -1) return existingPathIndex; - final newItem = CacheItem(value, - filePath: value.path, - previewHash: previewResult.previewHash, - fileSize: previewResult.fileSize)..addUse(); + filePath: value.path, + previewHash: previewResult.previewHash, + fileSize: previewResult.fileSize)..addUse(); _items.add(newItem); final index = _items.length - 1; _previewHashIndex[previewResult.previewHash] = index; // add to previously hashed return index; - } else if (value is FileImage) { final path = value.file.path; final File file = File(path); @@ -316,6 +401,16 @@ class AssetCacheAll { final existingHashIndex = _items.indexOf(newItem); if (existingHashIndex != -1) return existingHashIndex; + _items.add(newItem); + final index = _items.length - 1; + return index; + } else if (value is List) { // bytes + final hash = calculateHash(value); + final newItem = CacheItem(value, hash: hash)..addUse(); + + final existingHashIndex = _items.indexOf(newItem); + if (existingHashIndex != -1) return existingHashIndex; + _items.add(newItem); final index = _items.length - 1; return index; diff --git a/lib/components/canvas/image/pdf_editor_image.dart b/lib/components/canvas/image/pdf_editor_image.dart index 5a1bd196ea..6af498134c 100644 --- a/lib/components/canvas/image/pdf_editor_image.dart +++ b/lib/components/canvas/image/pdf_editor_image.dart @@ -1,6 +1,9 @@ part of 'editor_image.dart'; class PdfEditorImage extends EditorImage { + /// index of asset assigned to this pdf file + int assetId; + Uint8List? pdfBytes; final int pdfPage; @@ -16,6 +19,7 @@ class PdfEditorImage extends EditorImage { required super.id, required super.assetCache, required super.assetCacheAll, + required this.assetId, required this.pdfBytes, required this.pdfFile, required this.pdfPage, @@ -51,16 +55,17 @@ class PdfEditorImage extends EditorImage { String? extension = json['e'] as String?; assert(extension == null || extension == '.pdf'); - final assetIndex = json['a'] as int?; + final assetIndexJson = json['a'] as int?; final Uint8List? pdfBytes; + int? assetIndex; File? pdfFile; - if (assetIndex != null) { + if (assetIndexJson != null) { if (inlineAssets == null) { pdfFile = - FileManager.getFile('$sbnPath${Editor.extension}.$assetIndex'); + FileManager.getFile('$sbnPath${Editor.extension}.$assetIndexJson'); pdfBytes = assetCache.get(pdfFile); } else { - pdfBytes = inlineAssets[assetIndex]; + pdfBytes = inlineAssets[assetIndexJson]; } } else { if (kDebugMode) { @@ -69,11 +74,26 @@ class PdfEditorImage extends EditorImage { pdfBytes = Uint8List(0); } + assert(pdfBytes != null || pdfFile != null, + 'Either pdfBytes or pdfFile must be non-null'); + + // add to asset cache + if (pdfFile != null) { + assetIndex = assetCacheAll.addSync(pdfFile); + } + else { + assetIndex = assetCacheAll.addSync(pdfBytes!); + } + if (assetIndex<0){ + throw Exception('EditorImage.fromJson: pdf image not in assets'); + } + return PdfEditorImage( id: json['id'] ?? -1, // -1 will be replaced by EditorCoreInfo._handleEmptyImageIds() assetCache: assetCache, assetCacheAll: assetCacheAll, + assetId: assetIndex, pdfBytes: pdfBytes, pdfFile: pdfFile, pdfPage: json['pdfi'], @@ -130,9 +150,10 @@ class PdfEditorImage extends EditorImage { dstRect = dstRect.topLeft & dstSize; } - _pdfDocument.value ??= pdfFile != null - ? await PdfDocument.openFile(pdfFile!.path) - : await PdfDocument.openData(pdfBytes!); + _pdfDocument.value ??= await assetCacheAll.getPdfDocument(assetId); +// _pdfDocument.value ??= pdfFile != null +// ? await PdfDocument.openFile(pdfFile!.path) +// : await PdfDocument.openData(pdfBytes!); } @override @@ -188,6 +209,7 @@ class PdfEditorImage extends EditorImage { id: id, assetCache: assetCache, assetCacheAll: assetCacheAll, + assetId: assetId, pdfBytes: pdfBytes, pdfPage: pdfPage, pdfFile: pdfFile, diff --git a/lib/components/canvas/image/png_editor_image.dart b/lib/components/canvas/image/png_editor_image.dart index c27e51a770..573012909a 100644 --- a/lib/components/canvas/image/png_editor_image.dart +++ b/lib/components/canvas/image/png_editor_image.dart @@ -1,8 +1,14 @@ part of 'editor_image.dart'; class PngEditorImage extends EditorImage { + /// index of asset assigned to this image + int assetId; + + ImageProvider? imageProvider; + Future get imageProvider2 async => await assetCacheAll.getImageProvider(assetId); + Uint8List? thumbnailBytes; Size thumbnailSize = Size.zero; @@ -25,6 +31,7 @@ class PngEditorImage extends EditorImage { required super.id, required super.assetCache, required super.assetCacheAll, + required this.assetId, required super.extension, required this.imageProvider, required super.pageIndex, @@ -53,15 +60,13 @@ class PngEditorImage extends EditorImage { required AssetCacheAll assetCacheAll, }) { final assetIndexJson = json['a'] as int?; - final Uint8List? bytes; + Uint8List? bytes; final int? assetIndex; File? imageFile; if (assetIndexJson != null) { if (inlineAssets == null) { imageFile = FileManager.getFile('$sbnPath${Editor.extension}.$assetIndexJson'); - assetIndex=assetCacheAll.addSync(imageFile); - bytes = assetCache.get(imageFile); } else { bytes = inlineAssets[assetIndexJson]; } @@ -76,13 +81,25 @@ class PngEditorImage extends EditorImage { assert(bytes != null || imageFile != null, 'Either bytes or imageFile must be non-null'); + // add to asset cache + if (imageFile != null) { + assetIndex = assetCacheAll.addSync(imageFile); + } + else { + assetIndex = assetCacheAll.addSync(bytes!); + } + if (assetIndex<0){ + throw Exception('EditorImage.fromJson: image not in assets'); + } + return PngEditorImage( // -1 will be replaced by [EditorCoreInfo._handleEmptyImageIds()] id: json['id'] ?? -1, assetCache: assetCache, assetCacheAll: assetCacheAll, + assetId: assetIndex, extension: json['e'] ?? '.jpg', - imageProvider: bytes != null + imageProvider: bytes != null ? MemoryImage(bytes) as ImageProvider : FileImage(imageFile!), pageIndex: json['i'] ?? 0, @@ -225,6 +242,7 @@ class PngEditorImage extends EditorImage { id: id, assetCache: assetCache, assetCacheAll: assetCacheAll, + assetId: assetId, extension: extension, imageProvider: imageProvider, pageIndex: pageIndex, diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index c04da993dc..df0f7df6bd 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -1116,6 +1116,7 @@ class EditorState extends State { onLoad: () => setState(() {}), assetCache: coreInfo.assetCache, assetCacheAll: coreInfo.assetCacheAll, + assetId: cacheId, )); } } @@ -1196,6 +1197,8 @@ class EditorState extends State { log.severe('Failed to read file when importing $path: $e', e); return false; } + int? assetIndex = await coreInfo.assetCacheAll.add(pdfBytes); // add pdf to cache + final emptyPage = coreInfo.pages.removeLast(); assert(emptyPage.isEmpty); @@ -1234,6 +1237,7 @@ class EditorState extends State { onLoad: () => setState(() {}), assetCache: coreInfo.assetCache, assetCacheAll: coreInfo.assetCacheAll, + assetId: assetIndex, ); coreInfo.pages.add(page); history.recordChange(EditorHistoryItem( diff --git a/test/isolate_message_test.dart b/test/isolate_message_test.dart index 7bfabe7e7f..ec8527203a 100644 --- a/test/isolate_message_test.dart +++ b/test/isolate_message_test.dart @@ -37,6 +37,7 @@ void main() { onDeleteImage: null, onMiscChange: null, assetCache: coreInfo.assetCache, + assetCacheAll: coreInfo.assetCacheAll, ), ], ), @@ -60,6 +61,8 @@ void main() { onDeleteImage: null, onMiscChange: null, assetCache: coreInfo.assetCache, + assetCacheAll: coreInfo.assetCacheAll, + assetId: -1, ), ], ), @@ -83,6 +86,8 @@ void main() { onDeleteImage: null, onMiscChange: null, assetCache: coreInfo.assetCache, + assetCacheAll: coreInfo.assetCacheAll, + assetId: -1, ), ], ), diff --git a/test/tools_select_test.dart b/test/tools_select_test.dart index eebdc4d049..e75a102999 100644 --- a/test/tools_select_test.dart +++ b/test/tools_select_test.dart @@ -106,6 +106,7 @@ void main() { // ignore: missing_override_of_must_be_overridden class TestImage extends PngEditorImage { static final _assetCache = AssetCache(); + static final _assetCacheAll = AssetCacheAll(); TestImage({ required super.dstRect, @@ -119,6 +120,8 @@ class TestImage extends PngEditorImage { onDeleteImage: null, onMiscChange: null, assetCache: _assetCache, + assetCacheAll:_assetCacheAll, + assetId: -1, ); @override From 0b452e00ddbdf3de5d1b98f4aa42b547ec7c71e0 Mon Sep 17 00:00:00 2001 From: QubaB Date: Sun, 28 Sep 2025 08:13:14 +0200 Subject: [PATCH 03/35] Implemented image provider for png. --- lib/components/canvas/_asset_cache.dart | 244 ++++++++++++++---- .../canvas/image/png_editor_image.dart | 15 +- lib/pages/editor/editor.dart | 21 +- 3 files changed, 214 insertions(+), 66 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index 7ac75cb247..12d1708079 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -1,10 +1,12 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'dart:math'; import 'dart:typed_data'; import 'package:collection/collection.dart'; import 'package:flutter/painting.dart'; import 'package:logging/logging.dart'; +import 'package:path_provider/path_provider.dart'; import 'package:pdfrx/pdfrx.dart'; import 'package:saber/components/canvas/image/editor_image.dart'; @@ -147,14 +149,6 @@ class PreviewResult { PreviewResult(this.previewHash, this.fileSize); } - -// combine preview hash and size to "preliminary hash" -int makeCompositeKey(int previewHash, int fileSize) { - // Shift size into high bits, preview hash stays in low bits - // -> reduces collisions compared to just XOR - return (fileSize.hashCode << 32) ^ previewHash; -} - // object in cache class CacheItem { final Object value; @@ -166,12 +160,12 @@ class CacheItem { // for files only final int? fileSize; final String? filePath; // only for files - for fast comparison without reading file contents - + final String? fileExt; // file extension int _refCount = 0; // number of references bool _released = false; CacheItem(this.value, - {this.hash, this.filePath, this.previewHash, this.fileSize}); + {this.hash, this.filePath, this.previewHash, this.fileSize, this.fileExt}); // increase use of item @@ -214,7 +208,20 @@ class CacheItem { return false; // consider not equal } + @override + int get hashCode { + // If hash is not null, prefer it (since you compare on it first in ==) + if (hash != null) return hash.hashCode; + + // If previewHash is available, use it + if (previewHash != null) return previewHash.hashCode; + // Otherwise fall back to filePath + if (filePath != null) return filePath.hashCode; + + // Default fallback + return 0; + } // @override @@ -231,7 +238,6 @@ class AssetCacheAll { final Map _aliasMap = {}; // duplicit indices point to first indice - updated in finalize final Map _previewHashIndex = {}; // Map from previewHash → first index in _items - final Map> _pendingProviders = {}; // holds currently opening futures to avoid duplicates final Map> _openingDocs = {}; // Holds currently opening futures to avoid duplicate opens @@ -272,45 +278,27 @@ class AssetCacheAll { } } - Future getImageProvider(int assetId) { + // give image provider for asset image + ImageProvider getImageProvider(int assetId) { // return cached provider if already available final item = _items[assetId]; - if (item._imageProvider != null) return Future.value(item._imageProvider!); - - // if someone else is already creating it, return the pending future - final pending = _pendingProviders[assetId]; - if (pending != null) return pending; - - // otherwise, create the future - final future = _createImageProvider(item); - _pendingProviders[assetId] = future; - - // when done, store the provider and remove from pending - future.then((provider) { - item._imageProvider = provider; - _pendingProviders.remove(assetId); - }); + if (item._imageProvider != null) return item._imageProvider!; - return future; - } - - Future _createImageProvider(CacheItem item) async { if (item.value is File) { - return FileImage(item.value as File); + item._imageProvider = FileImage(item.value as File); + return item._imageProvider!; } else if (item.value is Uint8List) { - return MemoryImage(item.value as Uint8List); + item._imageProvider = MemoryImage(item.value as Uint8List); + return item._imageProvider!; } else if (item.value is MemoryImage) { return item.value as MemoryImage; } else if (item.value is FileImage) { return item.value as FileImage; } else { - throw UnsupportedError("Unsupported type for ImageProvider: ${item.value.runtimeType}"); + throw UnsupportedError('Unsupported type for ImageProvider: ${item.value.runtimeType}'); } } - - - // calculate hash of bytes (all) int calculateHash(List bytes) { // fnv1a int hash = 0x811C9DC5; @@ -343,11 +331,13 @@ class AssetCacheAll { // add to cache but read only small part of files - used when reading note from disk // full hashes are established later + // is used during read of note when it is opened. + // everything from new notes sba2 is File! + // only old notes provide bytes instead File int addSync(Object value) { if (value is File) { log.info('allCache.addSync: value = $value'); final path = value.path; - final previewResult=getFilePreviewHash(value); // calculate preliminary hash of file final existingPathIndex = _items.indexWhere((i) => i.filePath == path); if (existingPathIndex != -1) { @@ -355,6 +345,7 @@ class AssetCacheAll { log.info('allCache.addSync: already in cache {$_items[existingPathIndex]._refCount}'); return existingPathIndex; } + final previewResult=getFilePreviewHash(value); // calculate preliminary hash of file // Check if already cached if (_previewHashIndex.containsKey(previewResult.previewHash)) { @@ -376,6 +367,12 @@ class AssetCacheAll { final File file = File(path); final previewResult=getFilePreviewHash(file); // calculate preliminary hash of file + final existingPathIndex = _items.indexWhere((i) => i.filePath == path); + if (existingPathIndex != -1){ + _items[existingPathIndex].addUse(); + return existingPathIndex; + } + // Check if already cached if (_previewHashIndex.containsKey(previewResult.previewHash)) { final existingIndex = _previewHashIndex[previewResult.previewHash]!; @@ -383,8 +380,6 @@ class AssetCacheAll { return existingIndex; } - final existingPathIndex = _items.indexWhere((i) => i.filePath == path); - if (existingPathIndex != -1) return existingPathIndex; final newItem = CacheItem(value, filePath: path, @@ -428,7 +423,8 @@ class AssetCacheAll { } } - +// is used from Editor, when adding asset using file picker +// always is used File! Future add(Object value) async { if (value is File) { // files are first compared by file path final path = value.path; @@ -436,28 +432,59 @@ class AssetCacheAll { // 1. Fast path check final existingPathIndex = _items.indexWhere((i) => i.filePath == path); - if (existingPathIndex != -1) return existingPathIndex; + if (existingPathIndex != -1) { + _items[existingPathIndex].addUse(); + return existingPathIndex; + } + final previewResult=getFilePreviewHash(value); // calculate preliminary hash of file - // 2. Otherwise compute expensive content hash + // Check if already cached + if (_previewHashIndex.containsKey(previewResult.previewHash)) { + final existingIndex = _previewHashIndex[previewResult.previewHash]!; + _items[existingIndex].addUse(); + return existingIndex; + } + + // compute expensive content hash final bytes = await value.readAsBytes(); final hash = calculateHash(bytes); - final newItem = CacheItem(value, hash: hash, filePath: path); + final newItem = CacheItem(value, + filePath: value.path, + previewHash: previewResult.previewHash, + hash: hash, + fileSize: previewResult.fileSize)..addUse(); final existingHashIndex = _items.indexOf(newItem); - if (existingHashIndex != -1) return existingHashIndex; - + if (existingHashIndex != -1){ + _items[existingHashIndex].addUse(); + return existingHashIndex; + } _items.add(newItem); - return _items.length - 1; + final index = _items.length - 1; + _previewHashIndex[previewResult.previewHash] = index; // add to previously hashed + return index; } else if (value is FileImage) { // file images are first compared by file path final path = value.file.path; // 1. Fast path check final existingPathIndex = _items.indexWhere((i) => i.filePath == path); - if (existingPathIndex != -1) return existingPathIndex; + if (existingPathIndex != -1) { + _items[existingPathIndex].addUse(); + return existingPathIndex; + } + + final File file = File(path); + final previewResult=getFilePreviewHash(file); // calculate preliminary hash of file + + // Check if already cached + if (_previewHashIndex.containsKey(previewResult.previewHash)) { + final existingIndex = _previewHashIndex[previewResult.previewHash]!; + _items[existingIndex].addUse(); + return existingIndex; + } - // 2. Otherwise compute expensive content hash final bytes = await value.file.readAsBytes(); final hash = calculateHash(bytes); @@ -542,11 +569,128 @@ class AssetCacheAll { } } - /// Vrátí reálný index (přes alias mapu) + /// retunr real index through alias map int resolveIndex(int index) { return _aliasMap[index] ?? index; } + // generate random file name + String generateRandomFileName([String extension = 'txt']) { + final timestamp = DateTime.now().millisecondsSinceEpoch; + final random = Random().nextInt(1 << 32); // 32-bit random number + return 'file_${timestamp}_$random.$extension'; + } + + // create temporary file from bytes when inline bytes are read + Future createRuntimeFile(String ext, Uint8List bytes) async { + final dir = await getApplicationSupportDirectory(); + final file = File('${dir.path}/TmPmP_${generateRandomFileName(ext)}'); + return await file.writeAsBytes(bytes, flush: true); + } + + String readPdfMetadataSync(File file) { + if (!file.existsSync()) { + log.info('File not found'); + return ''; + } + + final fileSize = file.lengthSync(); + const trailerReadSize = 4096; + + final trailerStart = fileSize > trailerReadSize ? fileSize - trailerReadSize : 0; + final trailerLength = fileSize > trailerReadSize ? trailerReadSize : fileSize; + + final raf = file.openSync(); + final trailerBytes = Uint8List(trailerLength); + raf.setPositionSync(trailerStart); + raf.readIntoSync(trailerBytes); + raf.closeSync(); + + final trailerContent = ascii.decode(trailerBytes, allowInvalid: true); + final trailerIndex = trailerContent.lastIndexOf('trailer'); + if (trailerIndex == -1) { + log.info('Trailer not found'); + return ''; + } + + final trailerSlice = trailerContent.substring(trailerIndex); + final infoMatch = RegExp(r'/Info\s+(\d+)\s+(\d+)\s+R').firstMatch(trailerSlice); + if (infoMatch == null) { + log.info('Info object not found in trailer'); + return ''; + } + + final infoObjNumber = int.parse(infoMatch.group(1)!); + final infoObjGen = int.parse(infoMatch.group(2)!); + log.info('Info object reference: $infoObjNumber $infoObjGen'); + + // Find startxref + final startxrefIndex = trailerContent.lastIndexOf('startxref'); + if (startxrefIndex == -1) { + log.info('startxref not found'); + raf.closeSync(); + return ''; + } + + final startxrefLine = trailerContent.substring(startxrefIndex).split(RegExp(r'\r?\n'))[1].trim(); + final xrefOffset = int.tryParse(startxrefLine); + if (xrefOffset == null) { + log.info('Invalid startxref value'); + raf.closeSync(); + return ''; + } + + // Go to xref table + raf.setPositionSync(xrefOffset); + final xrefBuffer = Uint8List(8192); // read enough bytes for xref table + final bytesRead = raf.readIntoSync(xrefBuffer); + final xrefContent = ascii.decode(xrefBuffer.sublist(0, bytesRead), allowInvalid: true); + + // Find object offset in xref table + final objPattern = RegExp(r'(\d{10})\s+(\d{5})\s+n'); + final objMatches = objPattern.allMatches(xrefContent); + int? objOffset; + + int currentObjNumber = 0; + final subMatches = RegExp(r'(\d+)\s+(\d+)\s+(\d+)\s+n').allMatches(xrefContent); + for (final m in subMatches) { + final objNum = int.parse(m.group(1)!); + final offset = int.parse(m.group(2)!); + if (objNum == infoObjNumber) { + objOffset = offset; + break; + } + } + + if (objOffset == null) { + print('Info object offset not found in xref table'); + raf.closeSync(); + return ''; + } + + // Read Info object directly + raf.setPositionSync(objOffset); + final objBytes = Uint8List(1024); // read enough for object + final objRead = raf.readIntoSync(objBytes); + final objContent = ascii.decode(objBytes.sublist(0, objRead), allowInvalid: true); + + final objMatchContent = RegExp(r'obj(.*?)endobj', dotAll: true).firstMatch(objContent); + if (objMatchContent == null) { + log.info('Info object content not found'); + raf.closeSync(); + return ''; + } + + final infoContent = objMatchContent.group(1)!; + + final titleMatch = RegExp(r'/Title\s+\((.*?)\)').firstMatch(infoContent); + final authorMatch = RegExp(r'/Author\s+\((.*?)\)').firstMatch(infoContent); + final creationMatch = + RegExp(r'/CreationDate\s+\((.*?)\)').firstMatch(infoContent); + final String metadata='${titleMatch?.group(1) ?? 'N/A'}${authorMatch?.group(1) ?? 'N/A'}${creationMatch?.group(1) ?? 'N/A'}'; + return metadata; + } + @override String toString() => _items.toString(); } diff --git a/lib/components/canvas/image/png_editor_image.dart b/lib/components/canvas/image/png_editor_image.dart index 573012909a..3d957cfbbc 100644 --- a/lib/components/canvas/image/png_editor_image.dart +++ b/lib/components/canvas/image/png_editor_image.dart @@ -4,11 +4,7 @@ class PngEditorImage extends EditorImage { /// index of asset assigned to this image int assetId; - ImageProvider? imageProvider; - - Future get imageProvider2 async => await assetCacheAll.getImageProvider(assetId); - Uint8List? thumbnailBytes; Size thumbnailSize = Size.zero; @@ -86,12 +82,15 @@ class PngEditorImage extends EditorImage { assetIndex = assetCacheAll.addSync(imageFile); } else { - assetIndex = assetCacheAll.addSync(bytes!); + final tempFile=assetCacheAll.createRuntimeFile(json['e'] ?? '.jpg',bytes!); + assetIndex = assetCacheAll.addSync(tempFile); } if (assetIndex<0){ throw Exception('EditorImage.fromJson: image not in assets'); } + + return PngEditorImage( // -1 will be replaced by [EditorCoreInfo._handleEmptyImageIds()] id: json['id'] ?? -1, @@ -99,9 +98,7 @@ class PngEditorImage extends EditorImage { assetCacheAll: assetCacheAll, assetId: assetIndex, extension: json['e'] ?? '.jpg', - imageProvider: bytes != null - ? MemoryImage(bytes) as ImageProvider - : FileImage(imageFile!), + imageProvider: assetCacheAll.getImageProvider(assetIndex), pageIndex: json['i'] ?? 0, pageSize: Size.infinite, invertible: json['v'] ?? true, @@ -146,6 +143,7 @@ class PngEditorImage extends EditorImage { assert(Isolate.current.debugName == 'main'); if (srcRect.shortestSide == 0 || dstRect.shortestSide == 0) { + // when image was picked, its size is not determined. Do it final Uint8List bytes; if (imageProvider is MemoryImage) { bytes = (imageProvider as MemoryImage).bytes; @@ -175,6 +173,7 @@ class PngEditorImage extends EditorImage { height: reducedSize.height.toInt(), ); if (resizedByteData != null) { + imageProvider = MemoryImage(resizedByteData.buffer.asUint8List()); } diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index df0f7df6bd..6aed86f5d1 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -12,6 +12,7 @@ import 'package:flutter/services.dart'; import 'package:flutter_quill/flutter_quill.dart' as flutter_quill; import 'package:keybinder/keybinder.dart'; import 'package:logging/logging.dart'; +import 'package:path_provider/path_provider.dart'; import 'package:printing/printing.dart'; import 'package:saber/components/canvas/_asset_cache.dart'; import 'package:saber/components/canvas/_stroke.dart'; @@ -52,7 +53,7 @@ import 'package:saber/pages/home/whiteboard.dart'; import 'package:screenshot/screenshot.dart'; import 'package:super_clipboard/super_clipboard.dart'; -typedef _PhotoInfo = ({Uint8List bytes, String extension}); +typedef _PhotoInfo = ({Uint8List bytes, String extension, String path}); class Editor extends StatefulWidget { Editor({ @@ -1084,9 +1085,11 @@ class EditorState extends State { final List images = []; for (final _PhotoInfo photoInfo in photoInfos) { + + if (photoInfo.extension == '.svg') { - // add photo to cache - int cacheId = await coreInfo.assetCacheAll.add(photoInfo.bytes); + // add image to assets using its path + int assetIndex = await coreInfo.assetCacheAll.add(File(photoInfo.path)); images.add(SvgEditorImage( id: coreInfo.nextImageId++, svgString: utf8.decode(photoInfo.bytes), @@ -1102,12 +1105,12 @@ class EditorState extends State { )); } else { - final mImage=MemoryImage(photoInfo.bytes); - int cacheId = await coreInfo.assetCacheAll.add(mImage); + // add image to assets using its path + int assetIndex = await coreInfo.assetCacheAll.add(File(photoInfo.path)); images.add(PngEditorImage( id: coreInfo.nextImageId++, extension: photoInfo.extension, - imageProvider: MemoryImage(photoInfo.bytes), + imageProvider: coreInfo.assetCacheAll.getImageProvider(assetIndex), pageIndex: currentPageIndex, pageSize: coreInfo.pages[currentPageIndex].size, onMoveImage: onMoveImage, @@ -1116,7 +1119,7 @@ class EditorState extends State { onLoad: () => setState(() {}), assetCache: coreInfo.assetCache, assetCacheAll: coreInfo.assetCacheAll, - assetId: cacheId, + assetId: assetIndex, )); } } @@ -1166,6 +1169,7 @@ class EditorState extends State { ( bytes: file.bytes!, extension: '.${file.extension}', + path: file.path!, ), ]; } @@ -1197,7 +1201,7 @@ class EditorState extends State { log.severe('Failed to read file when importing $path: $e', e); return false; } - int? assetIndex = await coreInfo.assetCacheAll.add(pdfBytes); // add pdf to cache + int? assetIndex = await coreInfo.assetCacheAll.add(pdfFile); // add pdf to cache final emptyPage = coreInfo.pages.removeLast(); @@ -1308,6 +1312,7 @@ class EditorState extends State { photoInfos.add(( bytes: Uint8List.fromList(bytes), extension: extension, + path: file.fileName!, )); }, ); From 5171d15aa3ef8f904022a38a4e32ad5df9cc4ca7 Mon Sep 17 00:00:00 2001 From: QubaB Date: Sun, 28 Sep 2025 12:43:32 +0200 Subject: [PATCH 04/35] asset cache provides imageProvider as Notifier, so when more images share the same asset only one image provider is used replaceImage is implemented when png image is resized due to its greater size --- lib/components/canvas/_asset_cache.dart | 105 ++++++++++++++---- .../canvas/image/png_editor_image.dart | 57 +++++++--- lib/pages/editor/editor.dart | 2 +- 3 files changed, 122 insertions(+), 42 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index 12d1708079..1564e11a48 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -2,8 +2,8 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; import 'dart:math'; -import 'dart:typed_data'; import 'package:collection/collection.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:logging/logging.dart'; import 'package:path_provider/path_provider.dart'; @@ -154,7 +154,7 @@ class CacheItem { final Object value; int? previewHash; // quick hash (from first 100KB bytes) int? hash; // hash can be calculated later - ImageProvider? _imageProvider; // image provider for png, svg + final ValueNotifier imageProviderNotifier; // image provider for png, svg as value listener PdfDocument? _pdfDocument; // pdf document provider for pdf // for files only @@ -165,7 +165,13 @@ class CacheItem { bool _released = false; CacheItem(this.value, - {this.hash, this.filePath, this.previewHash, this.fileSize, this.fileExt}); + {this.hash, + this.filePath, + this.previewHash, + this.fileSize, + this.fileExt, + ValueNotifier? imageProviderNotifier, + }): imageProviderNotifier = imageProviderNotifier ?? ValueNotifier(null); // increase use of item @@ -191,6 +197,12 @@ class CacheItem { if (hash != null && other.hash != null) { return hash == other.hash; } + if (filePath != null && other.filePath != null) { + if (filePath == other.filePath) { + // both file paths are the same + return true; + } + } // Quick check using previewHash if (previewHash != null && other.previewHash != null) { @@ -199,12 +211,6 @@ class CacheItem { return false; } } - if (filePath != null && other.filePath != null) { - if (filePath == other.filePath) { - // both file paths are the same - return true; - } - } return false; // consider not equal } @@ -223,6 +229,27 @@ class CacheItem { return 0; } + // give image provider + ImageProvider getImageProvider(Object item) { + // return cached provider if already available + if (item is File) { + return FileImage(item); + } else if (item is Uint8List) { + return MemoryImage(item); + } else if (item is MemoryImage) { + return item; + } else if (item is FileImage) { + return item; + } else { + throw UnsupportedError('Unsupported type for ImageProvider: ${item.runtimeType}'); + } + } + + // invalidate image provider notifier value - called in case of image replacement + // this causes that new imageProvider will be created + void invalidateImageProvider() { + imageProviderNotifier.value = null; // will be recreated on next access + } // @override // int? get hash => filePath?.hash ?? hash; @@ -278,27 +305,30 @@ class AssetCacheAll { } } - // give image provider for asset image - ImageProvider getImageProvider(int assetId) { + // give image provider notifier for asset image + ValueNotifier getImageProviderNotifier(int assetId) { // return cached provider if already available final item = _items[assetId]; - if (item._imageProvider != null) return item._imageProvider!; + if (item.imageProviderNotifier.value != null) return item.imageProviderNotifier; if (item.value is File) { - item._imageProvider = FileImage(item.value as File); - return item._imageProvider!; + item.imageProviderNotifier.value = FileImage(item.value as File); } else if (item.value is Uint8List) { - item._imageProvider = MemoryImage(item.value as Uint8List); - return item._imageProvider!; + item.imageProviderNotifier.value = MemoryImage(item.value as Uint8List); } else if (item.value is MemoryImage) { - return item.value as MemoryImage; + item.imageProviderNotifier.value = item.value as MemoryImage; } else if (item.value is FileImage) { - return item.value as FileImage; + item.imageProviderNotifier.value = item.value as FileImage; } else { throw UnsupportedError('Unsupported type for ImageProvider: ${item.value.runtimeType}'); } + return item.imageProviderNotifier; } + + + + // calculate hash of bytes (all) int calculateHash(List bytes) { // fnv1a int hash = 0x811C9DC5; @@ -539,7 +569,7 @@ class AssetCacheAll { return item.file.readAsBytes(); } else { throw Exception( - 'OrderedAssetCache.getBytes: unknown type ${item.runtimeType}'); + 'assetCacheAll.getBytes: unknown type ${item.runtimeType}'); } } @@ -574,6 +604,35 @@ class AssetCacheAll { return _aliasMap[index] ?? index; } + // replace asset by another one - typically when resampling image to lower resolution + Future replaceImage(Object value,int id) async { + if (value is File) { + // compute expensive content hash + final bytes = await value.readAsBytes(); + final hash = calculateHash(bytes); + final previewResult=getFilePreviewHash(value); // calculate preliminary hash of file + + final oldItem=_items[id]; + // create new Cache item + final newItem = CacheItem(value, + filePath: value.path, + previewHash: previewResult.previewHash, + hash: hash, + fileSize: previewResult.fileSize, + imageProviderNotifier: oldItem.imageProviderNotifier, // keep original Notifier + ).._refCount=oldItem._refCount; // keep number of references + + // update original fields + _items[id] = newItem; + _items[id].invalidateImageProvider; // invalidate imageProvider so it is newly created when needed + } else { + throw Exception( + 'assetCacheAll.replaceImage: unknown type ${value.runtimeType}'); + } + } + + + // generate random file name String generateRandomFileName([String extension = 'txt']) { final timestamp = DateTime.now().millisecondsSinceEpoch; @@ -647,11 +706,11 @@ class AssetCacheAll { final xrefContent = ascii.decode(xrefBuffer.sublist(0, bytesRead), allowInvalid: true); // Find object offset in xref table - final objPattern = RegExp(r'(\d{10})\s+(\d{5})\s+n'); - final objMatches = objPattern.allMatches(xrefContent); + //final objPattern = RegExp(r'(\d{10})\s+(\d{5})\s+n'); + //final objMatches = objPattern.allMatches(xrefContent); int? objOffset; - int currentObjNumber = 0; + //nt currentObjNumber = 0; final subMatches = RegExp(r'(\d+)\s+(\d+)\s+(\d+)\s+n').allMatches(xrefContent); for (final m in subMatches) { final objNum = int.parse(m.group(1)!); @@ -663,7 +722,7 @@ class AssetCacheAll { } if (objOffset == null) { - print('Info object offset not found in xref table'); + log.info('Info object offset not found in xref table'); raf.closeSync(); return ''; } diff --git a/lib/components/canvas/image/png_editor_image.dart b/lib/components/canvas/image/png_editor_image.dart index 3d957cfbbc..812bfaecdd 100644 --- a/lib/components/canvas/image/png_editor_image.dart +++ b/lib/components/canvas/image/png_editor_image.dart @@ -4,7 +4,14 @@ class PngEditorImage extends EditorImage { /// index of asset assigned to this image int assetId; - ImageProvider? imageProvider; + // ImageProvider is given by assetCacheAll using this notifier + final ValueNotifier imageProviderNotifier; + + /// Convenience getter to access current ImageProvider + ImageProvider? get imageProvider => imageProviderNotifier.value; + + + Uint8List? thumbnailBytes; Size thumbnailSize = Size.zero; @@ -16,10 +23,11 @@ class PngEditorImage extends EditorImage { set isThumbnail(bool isThumbnail) { super.isThumbnail = isThumbnail; if (isThumbnail && thumbnailBytes != null) { - imageProvider = MemoryImage(thumbnailBytes!); - final scale = thumbnailSize.width / naturalSize.width; - srcRect = Rect.fromLTWH(srcRect.left * scale, srcRect.top * scale, - srcRect.width * scale, srcRect.height * scale); + // QBtodo - handle this thumbnail + //imageProvider = MemoryImage(thumbnailBytes!); + //final scale = thumbnailSize.width / naturalSize.width; + //srcRect = Rect.fromLTWH(srcRect.left * scale, srcRect.top * scale, + // srcRect.width * scale, srcRect.height * scale); } } @@ -29,7 +37,7 @@ class PngEditorImage extends EditorImage { required super.assetCacheAll, required this.assetId, required super.extension, - required this.imageProvider, + required this.imageProviderNotifier, required super.pageIndex, required super.pageSize, this.maxSize, @@ -98,7 +106,7 @@ class PngEditorImage extends EditorImage { assetCacheAll: assetCacheAll, assetId: assetIndex, extension: json['e'] ?? '.jpg', - imageProvider: assetCacheAll.getImageProvider(assetIndex), + imageProviderNotifier: assetCacheAll.getImageProviderNotifier(assetIndex), pageIndex: json['i'] ?? 0, pageSize: Size.infinite, invertible: json['v'] ?? true, @@ -173,8 +181,10 @@ class PngEditorImage extends EditorImage { height: reducedSize.height.toInt(), ); if (resizedByteData != null) { - - imageProvider = MemoryImage(resizedByteData.buffer.asUint8List()); + // store resized bytes to temporary file + final tempImageFile = await assetCacheAll.createRuntimeFile('.png',resizedByteData.buffer.asUint8List()); + // replace image + assetCacheAll.replaceImage(tempImageFile, assetId); } naturalSize = reducedSize; @@ -207,8 +217,10 @@ class PngEditorImage extends EditorImage { @override Future precache(BuildContext context) async { - if (imageProvider == null) return; - return await precacheImage(imageProvider!, context); + final provider = imageProviderNotifier.value; + if (provider != null) { + await precacheImage(provider, context); + } } @override @@ -227,12 +239,21 @@ class PngEditorImage extends EditorImage { boxFit = BoxFit.fill; } - return InvertWidget( - invert: invert, - child: Image( - image: imageProvider!, - fit: boxFit, - ), + return ValueListenableBuilder( + valueListenable: imageProviderNotifier, + builder: (context, provider, _) { + if (provider == null) { + return const SizedBox.shrink(); // nothing yet + } + + return InvertWidget( + invert: invert, + child: Image( + image: provider, + fit: boxFit, + ), + ); + }, ); } @@ -243,7 +264,7 @@ class PngEditorImage extends EditorImage { assetCacheAll: assetCacheAll, assetId: assetId, extension: extension, - imageProvider: imageProvider, + imageProviderNotifier: imageProviderNotifier, pageIndex: pageIndex, pageSize: Size.infinite, invertible: invertible, diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index 6aed86f5d1..8b6af95073 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -1110,7 +1110,7 @@ class EditorState extends State { images.add(PngEditorImage( id: coreInfo.nextImageId++, extension: photoInfo.extension, - imageProvider: coreInfo.assetCacheAll.getImageProvider(assetIndex), + imageProviderNotifier: coreInfo.assetCacheAll.getImageProviderNotifier(assetIndex), pageIndex: currentPageIndex, pageSize: coreInfo.pages[currentPageIndex].size, onMoveImage: onMoveImage, From 8023cf93719f89d826597e343e338f5edae32fe1 Mon Sep 17 00:00:00 2001 From: QubaB Date: Sun, 28 Sep 2025 14:01:43 +0200 Subject: [PATCH 05/35] added comment about new cache approach --- lib/components/canvas/_asset_cache.dart | 37 ++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index 1564e11a48..b4a7abbc74 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -140,6 +140,31 @@ class OrderedAssetCache { } /////////////////////////////////////////////////////////////////////////// /// New approach to cache +/// +/// current cache problems: +/// 1. two caches assetCache (for working), OrderedAssetCache (for writing) +/// 2. keeping bytes of each asset in memory (do not know if it is problem, but it is problem when adding to cache +/// because all bytes must be compared with each already added) +/// 3. after first saving of note containing pdf, each page is treated as different pdf +/// why: PdfEditorImage class +/// 1. keeps bytes = whole pdf +/// 2. creates its own pdfDocument renderer +/// 3. while saving note is to the OrderedAssetCache added each page of pdf separately as new asset. +/// when adding new items it s +/// +/// +/// handles jpg, png, pdf (not svg yet) +/// for each photo item provides ValueNotifier so the same items have the same provider +/// fore each pdf item provides PdfDocument every page of pdf use the same provider +/// +/// during reading note to Editor are new items added using addSync +/// +/// +/// +/// + + + // class returning preview data class PreviewResult { @@ -267,6 +292,10 @@ class AssetCacheAll { final Map> _openingDocs = {}; // Holds currently opening futures to avoid duplicate opens + /// Whether items from the cache can be removed: + /// set to false during file save. + bool allowRemovingAssets = true; + final log = Logger('OrderedAssetCache'); @@ -574,9 +603,9 @@ class AssetCacheAll { } // finalize cache after it was filled using addSync - without calculation of hashes + // is called after note is read to Editor Future finalize() async { final Map seenHashes = {}; // hash points to first index - for (int i = 0; i < _items.length; i++) { final item = _items[i]; int hash; @@ -752,4 +781,10 @@ class AssetCacheAll { @override String toString() => _items.toString(); + + void dispose() { + _items.clear(); + _cache.clear(); + } + } From c0e6886fcff39f32889cc0d62e79405882a623f8 Mon Sep 17 00:00:00 2001 From: QubaB Date: Sun, 28 Sep 2025 14:59:07 +0200 Subject: [PATCH 06/35] updated cache properties and description of problems of current cache --- lib/components/canvas/_asset_cache.dart | 45 ++++++++++++++++++------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index b4a7abbc74..45e90dc86f 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -139,29 +139,48 @@ class OrderedAssetCache { } } /////////////////////////////////////////////////////////////////////////// -/// New approach to cache -/// -/// current cache problems: +/// current cache and images problems: /// 1. two caches assetCache (for working), OrderedAssetCache (for writing) /// 2. keeping bytes of each asset in memory (do not know if it is problem, but it is problem when adding to cache /// because all bytes must be compared with each already added) -/// 3. after first saving of note containing pdf, each page is treated as different pdf +/// 3. after first saving of note importing pdf, is pdf saved as one asset because: +/// 1. importPdfFromFilePath create one File " final pdfFile = File(path);" +/// and this File is used to create all instances of pages +/// 2. when saving, all pages are one asset, because File is the same object!!! +/// +/// 4. when loading note again and saving note, each page is treated as different pdf /// why: PdfEditorImage class -/// 1. keeps bytes = whole pdf -/// 2. creates its own pdfDocument renderer -/// 3. while saving note is to the OrderedAssetCache added each page of pdf separately as new asset. -/// when adding new items it s +/// 1. when reading fromJson is created "pdfFile = FileManager.getFile('$sbnPath${Editor.extension}.$assetIndex');" +/// for each page (even if they are the same asset file) +/// 2. PdfEditorImage constructor is called with this File - each page has its own File!!! +/// 1. OrderedCache.add adds each page as new asset because each page is different File +/// 5. problems of PdfEditorImage +/// 1. PdfEditorImage keeps bytes of the whole pdf (wasting memory) even if it renders only one page +/// 2. creates its own pdfDocument renderer - for each pdf page is new renderer keeping whole pdf +/// 3. while saving note is to the OrderedAssetCache added each page of pdf separately as new asset. /// /// -/// handles jpg, png, pdf (not svg yet) -/// for each photo item provides ValueNotifier so the same items have the same provider -/// fore each pdf item provides PdfDocument every page of pdf use the same provider -/// -/// during reading note to Editor are new items added using addSync +/// New approach to cache /// +/// handles jpg, png, pdf (not svg yet) +/// for each photo item provides ValueNotifier so the same items have the same ImageProvider +/// for each pdf item provides PdfDocument every page of pdf use the same provider /// +/// During reading note to Editor are new items added using addSync - which is fast +/// addSync method: +/// 1. must treat duplicated assets (especially pdfs created by current OrderedCache) +/// 2. it calculate fast hash from first 100 KB of file and file size, if hash is the same files are "identical" +/// this is important only for compatibility. /// +/// In Editor is used async method when adding new image +/// add method: +/// it compares first paths, file size and then hashes of all cache items +/// calculation of hash is very time consuming, it will be better for pdfs to extract /Info and read author, creation date, etc. +/// and use this to recognize different pdfs. /// +/// Cache properties: +/// 1. Every cache item is created and treated as File (path). Even picked Photos are first saved as temporary files and then added to chache. +/// 2. Each item provides PdfDocument for pdfs or ValueNotifier for images. It saves memory From 9b67b1dee95e677b6c18dbb2916943cdf5f87a50 Mon Sep 17 00:00:00 2001 From: QubaB Date: Sun, 28 Sep 2025 16:49:14 +0200 Subject: [PATCH 07/35] added copyFile to FileManager - used to copy assets important is add $documentsDirectory to file name otherwise assets are saved to root of file system and it is not allowed --- lib/data/file_manager/file_manager.dart | 52 +++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/lib/data/file_manager/file_manager.dart b/lib/data/file_manager/file_manager.dart index e6d3949ecc..e03d83bd6b 100644 --- a/lib/data/file_manager/file_manager.dart +++ b/lib/data/file_manager/file_manager.dart @@ -240,6 +240,58 @@ class FileManager { if (awaitWrite) await writeFuture; } + /// Copies [fileFrom] to [filePath]. + /// + /// The file at [toPath] will have its last modified timestamp set to + /// [lastModified], if specified. + /// This is useful when downloading remote files, to make sure that the + /// timestamp is the same locally and remotely. + static Future copyFile( + File fileFrom, + String filePath, + { + bool awaitWrite = false, + bool alsoUpload = true, + DateTime? lastModified, + }) async { + filePath = _sanitisePath(filePath); + if (filePath.startsWith('/')){ + // cannot copy to root add document directory + filePath='$documentsDirectory$filePath'; + } + log.fine('Copying to $filePath'); + + await _saveFileAsRecentlyAccessed(filePath); + await _createFileDirectory(filePath); + final file = await fileFrom.copy(filePath); + Future writeFuture = Future.wait([ + if (lastModified != null) file.setLastModified(lastModified), + // if we're using a new format, also delete the old file + if (filePath.endsWith(Editor.extension)) + getFile( + '${filePath.substring(0, filePath.length - Editor.extension.length)}' + '${Editor.extensionOldJson}') + .delete() + // ignore if the file doesn't exist + .catchError((_) => File(''), + test: (e) => e is PathNotFoundException), + ]); + + void afterWrite() { + broadcastFileWrite(FileOperationType.write, filePath); + if (alsoUpload) syncer.uploader.enqueueRel(filePath); + if (filePath.endsWith(Editor.extension)) { + _removeReferences( + '${filePath.substring(0, filePath.length - Editor.extension.length)}' + '${Editor.extensionOldJson}'); + } + } + + writeFuture = writeFuture.then((_) => afterWrite()); + if (awaitWrite) await writeFuture; + } + + static Future createFolder(String folderPath) async { folderPath = _sanitisePath(folderPath); From efc1155d63b4602f3a78ab1f5d626fa526a92d46 Mon Sep 17 00:00:00 2001 From: QubaB Date: Sun, 28 Sep 2025 16:49:49 +0200 Subject: [PATCH 08/35] implemented saving of note using new assetCacheAll cache --- lib/components/canvas/_asset_cache.dart | 18 ++++++++++++----- lib/pages/editor/editor.dart | 26 +++++++++++++------------ 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index 45e90dc86f..a3e5c9334c 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -9,6 +9,7 @@ import 'package:logging/logging.dart'; import 'package:path_provider/path_provider.dart'; import 'package:pdfrx/pdfrx.dart'; import 'package:saber/components/canvas/image/editor_image.dart'; +import 'package:saber/data/file_manager/file_manager.dart'; /// A cache for assets that are loaded from disk. /// @@ -679,6 +680,18 @@ class AssetCacheAll { } } + // return File associated with asset, used to save assets when saving note + File getAssetFile(int id){ + final item = _items[id]; + if (item.value is File) { + return (item.value as File); + } else if (item is FileImage) { + return (item.value as FileImage).file; + } else { + throw Exception( + 'assetCacheAll.getBytes: unknown type ${item.runtimeType}'); + } + } // generate random file name @@ -801,9 +814,4 @@ class AssetCacheAll { @override String toString() => _items.toString(); - void dispose() { - _items.clear(); - _cache.clear(); - } - } diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index 8b6af95073..1ae47759bb 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -906,27 +906,29 @@ class EditorState extends State { final Uint8List bson; final OrderedAssetCache assets; coreInfo.assetCache.allowRemovingAssets = false; + coreInfo.assetCacheAll.allowRemovingAssets = false; try { + // go through all pages and prepare Json of each page. (bson, assets) = coreInfo.saveToBinary( currentPageIndex: currentPageIndex, ); } finally { coreInfo.assetCache.allowRemovingAssets = true; + coreInfo.assetCacheAll.allowRemovingAssets = true; } try { - await Future.wait([ - FileManager.writeFile(filePath, bson, awaitWrite: true), - for (int i = 0; i < assets.length; ++i) - assets.getBytes(i).then((bytes) => FileManager.writeFile( - '$filePath.$i', - bytes, - awaitWrite: true, - )), - FileManager.removeUnusedAssets( + // write note itself + await FileManager.writeFile(filePath, bson, awaitWrite: true); + + // write assets + for (int i = 0; i < coreInfo.assetCacheAll.length; ++i){ + final assetFile=coreInfo.assetCacheAll.getAssetFile(i); + await FileManager.copyFile(assetFile,'$filePath.$i', awaitWrite: true); + } + FileManager.removeUnusedAssets( filePath, - numAssets: assets.length, - ), - ]); + numAssets: coreInfo.assetCacheAll.length, + ); savingState.value = SavingState.saved; } catch (e) { log.severe('Failed to save file: $e', e); From 1f3f9831b207b02c6b012e661fe109ebdb89d7d2 Mon Sep 17 00:00:00 2001 From: QubaB Date: Sun, 28 Sep 2025 20:10:01 +0200 Subject: [PATCH 09/35] FileManager - reworked copyFile to add correct base directory --- lib/data/file_manager/file_manager.dart | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/data/file_manager/file_manager.dart b/lib/data/file_manager/file_manager.dart index e03d83bd6b..053884265e 100644 --- a/lib/data/file_manager/file_manager.dart +++ b/lib/data/file_manager/file_manager.dart @@ -189,6 +189,18 @@ class FileManager { } } + // return file path (add document directory if needed) + static String getFilePath(String filePath) { + if (shouldUseRawFilePath) { + return filePath; + } else { + assert(filePath.startsWith('/'), + 'Expected filePath to start with a slash, got $filePath'); + return '$documentsDirectory$filePath'; + } + } + + static Directory getRootDirectory() => Directory(documentsDirectory); /// Writes [toWrite] to [filePath]. @@ -255,14 +267,12 @@ class FileManager { DateTime? lastModified, }) async { filePath = _sanitisePath(filePath); - if (filePath.startsWith('/')){ - // cannot copy to root add document directory - filePath='$documentsDirectory$filePath'; - } + await _createFileDirectory(filePath); // create directory filePath is "relative to saber documents directory") + + filePath = getFilePath(filePath); // if needed add documents directory to file path to have full path log.fine('Copying to $filePath'); await _saveFileAsRecentlyAccessed(filePath); - await _createFileDirectory(filePath); final file = await fileFrom.copy(filePath); Future writeFuture = Future.wait([ if (lastModified != null) file.setLastModified(lastModified), From 0b5407b6e8547a205a6c391de26df72dd4e46437 Mon Sep 17 00:00:00 2001 From: QubaB Date: Sun, 5 Oct 2025 23:08:29 +0200 Subject: [PATCH 10/35] implemented PdfInfoExtractor which extracts from pdf information. It can be used to recognize the same pdf files instead of calculating hash of all file contents class parse linearized and normal pdfs. --- lib/components/canvas/_asset_cache.dart | 541 +++++++++++++++++++----- 1 file changed, 437 insertions(+), 104 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index a3e5c9334c..69e1c61302 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -11,6 +11,265 @@ import 'package:pdfrx/pdfrx.dart'; import 'package:saber/components/canvas/image/editor_image.dart'; import 'package:saber/data/file_manager/file_manager.dart'; +class PdfInfoExtractor { + /// Extracts /Info dictionary from a PDF file with minimal reads (synchronous) + static Map extractInfo(File file) { + final fileSize = file.lengthSync(); + final raf = file.openSync(); + + try { + // Check if linearized (optimized for web) by reading first 256 bytes + raf.setPositionSync(0); + final startBytes = raf.readSync(256); + final startContent = latin1.decode(startBytes); + + final isLinearized = startContent.contains('/Linearized'); + + String trailerContent; + int xrefOffset = 0; // offset of xref on the start of file + int xrefOffsetEnd = 0; // offset of xref on the end of file + int infoOffset=0; + int infoObjNum = 0; + if (isLinearized) { + raf.setPositionSync(0); + final linearizedBytes = raf.readSync(2048); + final startContent = latin1.decode(linearizedBytes); + + // Check if /Info is referenced in start trailer + // Look directly for /Info pattern in the trailer section + final infoMatch = RegExp( + r'trailer\s*<<.*/Info\s+(\d+)\s+\d+\s+', dotAll: true) + .firstMatch(startContent); + if (infoMatch != null && infoMatch.group(1) != null) { + // /Info reference is at start, but we need end xref for object offset + infoObjNum = int.parse(infoMatch.group(1)!); + } + + // try to find /Prev indicating where starts xref on the end of file + final prevMatch = RegExp(r'/Prev\s+(\d+)').firstMatch(startContent); + if (prevMatch != null && prevMatch.group(1) != null) { + // /Info reference is at start, but we need end xref for object offset + xrefOffsetEnd = int.parse(prevMatch.group(1)!); + } + + // try to find xref in the starting section + int xrefStart = 0; + int xrefSize = 0; + // Look directly for /Info pattern in the trailer section + final startxRefMatch = RegExp(r'xref\s*(\d+)\s+(\d+)', dotAll: true) + .firstMatch(startContent); + if (startxRefMatch != null && startxRefMatch.group(1) != null && + startxRefMatch.group(2) != null) { + xrefStart = int.parse(startxRefMatch.group(1)!); + xrefSize = int.parse(startxRefMatch.group(2)!); + } + + if (infoObjNum != 0 && xrefSize != 0 && infoObjNum >= xrefStart && + infoObjNum <= xrefStart + xrefSize) { + // Info object is referenced in xref on the document start, find its offset + infoOffset = _findObjectOffset(raf, 0, infoObjNum, fileSize,startContent); + if (infoOffset==-2) { + // error unexpected + } + } + else { + infoOffset=-1; // Info object is not in this xref + } + if (infoOffset==-1 && xrefOffsetEnd != 0) { + // info object is not in this xref table it is at the end of file at xrefOffsetEnd found in /Prev + infoOffset = _findObjectOffset(raf, xrefOffsetEnd, infoObjNum,fileSize, ""); + } + if (infoOffset<0) { + infoOffset = -2; + Map result = {}; + return result; // return empty info + } + } + if (infoOffset == 0) { + // standard pdf with Info at the end of file + // Extract Info object number from trailer + final endOffset = fileSize > 1024 ? fileSize - 1024 : 0; + raf.setPositionSync(endOffset); + final endBytes = raf.readSync(1024); + final endContent = latin1.decode(endBytes); + // Find trailer and extract /Info object number + final infoMatch = RegExp( + r'trailer\s*<<.*/Info\s+(\d+)\s+\d+\s+', dotAll: true) + .firstMatch(endContent); + if (infoMatch != null && infoMatch.group(1) != null) { + // /Info reference is at start, but we need end xref for object offset + infoObjNum = int.parse(infoMatch.group(1)!); + } + else { + throw Exception('Trailer not found'); + } + // Find startxref offset + final startxrefMatch = RegExp(r'startxref\s+(\d+)').firstMatch( + endContent); + if (startxrefMatch == null) { + throw Exception('startxref not found'); + } + xrefOffset = int.parse(startxrefMatch.group(1)!); + infoOffset = + _findObjectOffset(raf, xrefOffset, infoObjNum, fileSize, ""); + } + if (infoOffset != 0) { + return _extractInfoFromTrailer(raf, infoOffset, fileSize); + } + Map result = {}; + return result; + } finally { + raf.closeSync(); + } + } + + static int _findXrefFromEnd(RandomAccessFile raf, int fileSize) { + final endOffset = fileSize > 1024 ? fileSize - 1024 : 0; + raf.setPositionSync(endOffset); + final endBytes = raf.readSync(1024); + final endContent = latin1.decode(endBytes); + + final startxrefMatch = RegExp(r'startxref\s+(\d+)').firstMatch(endContent); + if (startxrefMatch == null) { + throw Exception('startxref not found'); + } + return int.parse(startxrefMatch.group(1)!); + } + + static Map _extractInfoFromTrailer( + RandomAccessFile raf, + int infoOffset, + int fileSize, + ) { + // Read Info object - read up to 4KB to handle large info dictionaries + raf.setPositionSync(infoOffset); + final bytesToRead = (fileSize - infoOffset) < 4096 + ? fileSize - infoOffset + : 4096; + final infoBytes = raf.readSync(bytesToRead); + final infoContent = latin1.decode(infoBytes); + // Parse Info dictionary + return _parseInfoDict(infoContent); + } + + + // find object objNum in xref table + static int _findObjectOffset( + RandomAccessFile raf, + int xrefOffset, + int objNum, + int fileSize, + String xrefContent, + ) { + // Parse xref table + if (xrefContent.isEmpty){ + // there is no content read, do it + // Read xref table to find object offset + raf.setPositionSync(xrefOffset); + final remainingBytes = fileSize - xrefOffset; + final xrefBytes = raf.readSync(remainingBytes); + xrefContent = latin1.decode(xrefBytes); + } + final xrefLines = xrefContent.split('\n'); + int startNum = 0; + int count = 0; + int i0=-1; + + int iStart=-1; + int iStartTable=0; + // find line xref + for (int i = 0; i < xrefLines.length; i++) { + final line = xrefLines[i].trim(); + if (line == 'xref') { + iStart=i+1; + } + break; + } + if (iStart<0){ + throw('xref section does not contain xref'); + } + while (iStart< xrefLines.length) { + final line = xrefLines[iStart].trim(); + final subsectionMatch = RegExp(r'^(\d+)\s+(\d+)$').firstMatch( + line.trim()); + if (subsectionMatch != null) { + startNum = int.parse(subsectionMatch.group(1)!); + count = int.parse(subsectionMatch.group(2)!); + iStart=iStart+1; // starting line of table + } + else { + throw('Error determining xref objects range'); + } + if (objNumstartNum+count){ + // object is not in this section of xref Table try another one + iStart=iStart+count; // try another section + } + else { + break; + } + } + // object is in the table and it is on position + int indexObj=iStart+(objNum-startNum); + if (indexObj>xrefLines.length) { + // not read enough number of lines + return -2; + } + // Parse entry (e.g., "0000000015 00000 n") + final entryMatch = + RegExp(r'^(\d{10})\s+(\d{5})\s+([nf])').firstMatch(xrefLines[indexObj].trim()); + if (entryMatch != null) { + return int.parse(entryMatch.group(1)!); + } + return -3; + } + + static Map _parseInfoDict(String content) { + final result = {}; + +// Find the Info dictionary between << and >> + final dictMatch = RegExp(r'<<([^>]*)>>', dotAll: true).firstMatch(content); + if (dictMatch == null) return result; + + final dictContent = dictMatch.group(1)!; + +// Extract key-value pairs + final entries = RegExp(r'/(\w+)\s*\(([^)]*)\)').allMatches(dictContent); + + for (final match in entries) { + final key = match.group(1)!; + var value = match.group(2)!; + +// Decode PDF string (basic implementation) + value = value + .replaceAll(r'\(', '(') + .replaceAll(r'\)', ')') + .replaceAll(r'\\', '\\') + .replaceAll(r'\n', '\n') + .replaceAll(r'\r', '\r') + .replaceAll(r'\t', '\t'); + + result[key] = value; + } + +// Also handle hex strings <...> + final hexEntries = + RegExp(r'/(\w+)\s*<([0-9A-Fa-f]+)>').allMatches(dictContent); + for (final match in hexEntries) { + final key = match.group(1)!; + final hexValue = match.group(2)!; + +// Convert hex to string + final bytes = []; + for (int i = 0; i < hexValue.length; i += 2) { + bytes.add(int.parse(hexValue.substring(i, i + 2), radix: 16)); + } + result[key] = utf8.decode(bytes, allowMalformed: true); + } + + return result; + } +} + /// A cache for assets that are loaded from disk. /// /// This is the analogue to Flutter's image cache, @@ -183,41 +442,43 @@ class OrderedAssetCache { /// 1. Every cache item is created and treated as File (path). Even picked Photos are first saved as temporary files and then added to chache. /// 2. Each item provides PdfDocument for pdfs or ValueNotifier for images. It saves memory - - - // class returning preview data class PreviewResult { final int previewHash; final int fileSize; - - PreviewResult(this.previewHash, this.fileSize); + final String firstBytes; + final String fileInfo; // string containing file information - now only pdfFile /Info + PreviewResult(this.previewHash, this.fileSize, this.firstBytes, this.fileInfo); } // object in cache class CacheItem { final Object value; int? previewHash; // quick hash (from first 100KB bytes) - int? hash; // hash can be calculated later - final ValueNotifier imageProviderNotifier; // image provider for png, svg as value listener - PdfDocument? _pdfDocument; // pdf document provider for pdf + int? hash; // hash can be calculated later + String? fileInfo; // file information - /Info of pdf is implemented now + final ValueNotifier + imageProviderNotifier; // image provider for png, svg as value listener + PdfDocument? _pdfDocument; // pdf document provider for pdf // for files only final int? fileSize; - final String? filePath; // only for files - for fast comparison without reading file contents - final String? fileExt; // file extension - int _refCount = 0; // number of references + final String? + filePath; // only for files - for fast comparison without reading file contents + final String? fileExt; // file extension + int _refCount = 0; // number of references bool _released = false; - CacheItem(this.value, - {this.hash, - this.filePath, - this.previewHash, - this.fileSize, - this.fileExt, - ValueNotifier? imageProviderNotifier, - }): imageProviderNotifier = imageProviderNotifier ?? ValueNotifier(null); - + CacheItem( + this.value, { + this.hash, + this.filePath, + this.previewHash, + this.fileSize, + this.fileExt, + this.fileInfo, + ValueNotifier? imageProviderNotifier, + }) : imageProviderNotifier = imageProviderNotifier ?? ValueNotifier(null); // increase use of item void addUse() { @@ -282,11 +543,12 @@ class CacheItem { } else if (item is Uint8List) { return MemoryImage(item); } else if (item is MemoryImage) { - return item; + return item; } else if (item is FileImage) { - return item; + return item; } else { - throw UnsupportedError('Unsupported type for ImageProvider: ${item.runtimeType}'); + throw UnsupportedError( + 'Unsupported type for ImageProvider: ${item.runtimeType}'); } } @@ -307,17 +569,18 @@ class CacheItem { // cache manager class AssetCacheAll { final List _items = []; - final Map _aliasMap = {}; // duplicit indices point to first indice - updated in finalize - final Map _previewHashIndex = {}; // Map from previewHash → first index in _items + final Map _aliasMap = + {}; // duplicit indices point to first indice - updated in finalize + final Map _previewHashIndex = + {}; // Map from previewHash → first index in _items - final Map> _openingDocs = {}; // Holds currently opening futures to avoid duplicate opens + final Map> _openingDocs = + {}; // Holds currently opening futures to avoid duplicate opens /// Whether items from the cache can be removed: /// set to false during file save. bool allowRemovingAssets = true; - - final log = Logger('OrderedAssetCache'); // return pdfDocument of asset it is lazy because it take some time to do it @@ -358,7 +621,8 @@ class AssetCacheAll { ValueNotifier getImageProviderNotifier(int assetId) { // return cached provider if already available final item = _items[assetId]; - if (item.imageProviderNotifier.value != null) return item.imageProviderNotifier; + if (item.imageProviderNotifier.value != null) + return item.imageProviderNotifier; if (item.value is File) { item.imageProviderNotifier.value = FileImage(item.value as File); @@ -369,17 +633,15 @@ class AssetCacheAll { } else if (item.value is FileImage) { item.imageProviderNotifier.value = item.value as FileImage; } else { - throw UnsupportedError('Unsupported type for ImageProvider: ${item.value.runtimeType}'); + throw UnsupportedError( + 'Unsupported type for ImageProvider: ${item.value.runtimeType}'); } return item.imageProviderNotifier; } - - - - // calculate hash of bytes (all) - int calculateHash(List bytes) { // fnv1a + int calculateHash(List bytes) { + // fnv1a int hash = 0x811C9DC5; for (var b in bytes) { hash ^= b; @@ -392,7 +654,7 @@ class AssetCacheAll { // This can be done synchronously to quickly filter duplicates. // calculate preview hash of file PreviewResult getFilePreviewHash(File file) { - final stat = file.statSync(); // get file metadata + final stat = file.statSync(); // get file metadata final fileSize = stat.size; final raf = file.openSync(mode: FileMode.read); @@ -401,8 +663,24 @@ class AssetCacheAll { final toRead = fileSize < 100 * 1024 ? fileSize : 100 * 1024; final bytes = raf.readSync(toRead); final previewHash = calculateHash(bytes); - return PreviewResult((fileSize.hashCode << 32) ^ previewHash, // hash - fileSize); // file size + final firstBytes =ascii.decode(bytes.sublist(0, 4)); // first 4 characters - used to detect PDF file + String fileInfo=""; + if (firstBytes == "%PDF") { + try { + final info = PdfInfoExtractor.extractInfo(file); + fileInfo=info.values.join(','); // convert file info to String + } + catch(e) { + fileInfo=""; + } + } + + return PreviewResult( + (fileSize.hashCode << 32) ^ previewHash, // hash + fileSize, // file size + firstBytes, // first 4 bytes - used to recognize pdf file format + fileInfo + ); } finally { raf.closeSync(); } @@ -420,13 +698,29 @@ class AssetCacheAll { final existingPathIndex = _items.indexWhere((i) => i.filePath == path); if (existingPathIndex != -1) { + // file path already in cache so file _items[existingPathIndex].addUse(); - log.info('allCache.addSync: already in cache {$_items[existingPathIndex]._refCount}'); + log.info( + 'allCache.addSync: already in cache {$_items[existingPathIndex]._refCount}'); return existingPathIndex; } - final previewResult=getFilePreviewHash(value); // calculate preliminary hash of file - // Check if already cached + final previewResult = + getFilePreviewHash(value); // calculate preliminary hash of file or get Info of pdf file + + if (previewResult.fileInfo.isNotEmpty){ + // we know information about file so test it + final existingFileInfoIndex = _items.indexWhere((i) => i.fileInfo == previewResult.fileInfo); + if (existingFileInfoIndex != -1) { + // file with this fileInfo already in cache + _items[existingFileInfoIndex].addUse(); + log.info( + 'allCache.addSync: already in cache {$_items[existingPathIndex]._refCount}'); + return existingFileInfoIndex; + } + } + + // Check if already cached previewHash if (_previewHashIndex.containsKey(previewResult.previewHash)) { final existingIndex = _previewHashIndex[previewResult.previewHash]!; _items[existingIndex].addUse(); @@ -436,18 +730,23 @@ class AssetCacheAll { final newItem = CacheItem(value, filePath: value.path, previewHash: previewResult.previewHash, - fileSize: previewResult.fileSize)..addUse(); + fileSize: previewResult.fileSize, + fileInfo: previewResult.fileInfo) + ..addUse(); // and add use + _items.add(newItem); final index = _items.length - 1; - _previewHashIndex[previewResult.previewHash] = index; // add to previously hashed + _previewHashIndex[previewResult.previewHash] = + index; // add to previously hashed return index; } else if (value is FileImage) { final path = value.file.path; final File file = File(path); - final previewResult=getFilePreviewHash(file); // calculate preliminary hash of file + final previewResult = + getFilePreviewHash(file); // calculate preliminary hash of file final existingPathIndex = _items.indexWhere((i) => i.filePath == path); - if (existingPathIndex != -1){ + if (existingPathIndex != -1) { _items[existingPathIndex].addUse(); return existingPathIndex; } @@ -459,16 +758,18 @@ class AssetCacheAll { return existingIndex; } - final newItem = CacheItem(value, - filePath: path, - previewHash: previewResult.previewHash, - fileSize: previewResult.fileSize)..addUse(); + filePath: path, + previewHash: previewResult.previewHash, + fileSize: previewResult.fileSize) + ..addUse(); _items.add(newItem); final index = _items.length - 1; - _previewHashIndex[previewResult.previewHash] = index; // add to previously hashed + _previewHashIndex[previewResult.previewHash] = + index; // add to previously hashed return index; - } else if (value is MemoryImage) { // file images are first compared by file path + } else if (value is MemoryImage) { + // file images are first compared by file path final hash = calculateHash(value.bytes); final newItem = CacheItem(value, hash: hash)..addUse(); @@ -478,7 +779,8 @@ class AssetCacheAll { _items.add(newItem); final index = _items.length - 1; return index; - } else if (value is List) { // bytes + } else if (value is List) { + // bytes final hash = calculateHash(value); final newItem = CacheItem(value, hash: hash)..addUse(); @@ -488,14 +790,14 @@ class AssetCacheAll { _items.add(newItem); final index = _items.length - 1; return index; - } else if (value is String){ - // directly calculate hash - final newItem = CacheItem(value, hash: value.hashCode)..addUse(); - final existingHashIndex = _items.indexOf(newItem); - if (existingHashIndex != -1) return existingHashIndex; - _items.add(newItem); - final index = _items.length - 1; - return index; + } else if (value is String) { + // directly calculate hash + final newItem = CacheItem(value, hash: value.hashCode)..addUse(); + final existingHashIndex = _items.indexOf(newItem); + if (existingHashIndex != -1) return existingHashIndex; + _items.add(newItem); + final index = _items.length - 1; + return index; } else { throw Exception( 'OrderedAssetCache.getBytes: unknown type ${value.runtimeType}'); @@ -505,17 +807,30 @@ class AssetCacheAll { // is used from Editor, when adding asset using file picker // always is used File! Future add(Object value) async { - if (value is File) { // files are first compared by file path + if (value is File) { + // files are first compared by file path final path = value.path; // 1. Fast path check - final existingPathIndex = - _items.indexWhere((i) => i.filePath == path); + final existingPathIndex = _items.indexWhere((i) => i.filePath == path); if (existingPathIndex != -1) { _items[existingPathIndex].addUse(); return existingPathIndex; } - final previewResult=getFilePreviewHash(value); // calculate preliminary hash of file + final previewResult = + getFilePreviewHash(value); // calculate preliminary hash of file + + if (previewResult.fileInfo.isNotEmpty){ + // we know information about file so test it + final existingFileInfoIndex = _items.indexWhere((i) => i.fileInfo == previewResult.fileInfo); + if (existingFileInfoIndex != -1) { + // file with this fileInfo already in cache + _items[existingFileInfoIndex].addUse(); + log.info( + 'allCache.addSync: already in cache {$_items[existingPathIndex]._refCount}'); + return existingFileInfoIndex; + } + } // Check if already cached if (_previewHashIndex.containsKey(previewResult.previewHash)) { @@ -532,30 +847,33 @@ class AssetCacheAll { filePath: value.path, previewHash: previewResult.previewHash, hash: hash, - fileSize: previewResult.fileSize)..addUse(); + fileSize: previewResult.fileSize) + ..addUse(); final existingHashIndex = _items.indexOf(newItem); - if (existingHashIndex != -1){ + if (existingHashIndex != -1) { _items[existingHashIndex].addUse(); return existingHashIndex; } _items.add(newItem); final index = _items.length - 1; - _previewHashIndex[previewResult.previewHash] = index; // add to previously hashed + _previewHashIndex[previewResult.previewHash] = + index; // add to previously hashed return index; - } else if (value is FileImage) { // file images are first compared by file path + } else if (value is FileImage) { + // file images are first compared by file path final path = value.file.path; // 1. Fast path check - final existingPathIndex = - _items.indexWhere((i) => i.filePath == path); + final existingPathIndex = _items.indexWhere((i) => i.filePath == path); if (existingPathIndex != -1) { _items[existingPathIndex].addUse(); return existingPathIndex; } final File file = File(path); - final previewResult=getFilePreviewHash(file); // calculate preliminary hash of file + final previewResult = + getFilePreviewHash(file); // calculate preliminary hash of file // Check if already cached if (_previewHashIndex.containsKey(previewResult.previewHash)) { @@ -574,7 +892,8 @@ class AssetCacheAll { _items.add(newItem); return _items.length - 1; - } else if (value is MemoryImage) { // file images are first compared by file path + } else if (value is MemoryImage) { + // file images are first compared by file path final hash = calculateHash(value.bytes); final newItem = CacheItem(value, hash: hash); @@ -585,7 +904,7 @@ class AssetCacheAll { _items.add(newItem); return _items.length - 1; } else { - final hash = value.hashCode; // string + final hash = value.hashCode; // string final newItem = CacheItem(value, hash: hash); final existingIndex = _items.indexOf(newItem); @@ -604,7 +923,8 @@ class AssetCacheAll { /// Converts the item at position [indexIn] /// to bytes and returns them. Future> getBytes(int indexIn) async { - final index = resolveIndex(indexIn); // find first occurence in cache to avoid duplicities + final index = resolveIndex( + indexIn); // find first occurence in cache to avoid duplicities final item = _items[index].value; if (item is List) { return item; @@ -633,10 +953,10 @@ class AssetCacheAll { if (hashItem == 0) { final bytes = await getBytes(i); hash = calculateHash(bytes); - _items[i] = CacheItem(item.value, hash: hash, filePath: item.filePath, fileSize: item.fileSize); - } - else { - hash=hashItem!; + _items[i] = CacheItem(item.value, + hash: hash, filePath: item.filePath, fileSize: item.fileSize); + } else { + hash = hashItem!; } if (seenHashes.containsKey(hash)) { @@ -654,34 +974,38 @@ class AssetCacheAll { } // replace asset by another one - typically when resampling image to lower resolution - Future replaceImage(Object value,int id) async { + Future replaceImage(Object value, int id) async { if (value is File) { // compute expensive content hash final bytes = await value.readAsBytes(); final hash = calculateHash(bytes); - final previewResult=getFilePreviewHash(value); // calculate preliminary hash of file + final previewResult = + getFilePreviewHash(value); // calculate preliminary hash of file - final oldItem=_items[id]; + final oldItem = _items[id]; // create new Cache item - final newItem = CacheItem(value, - filePath: value.path, - previewHash: previewResult.previewHash, - hash: hash, - fileSize: previewResult.fileSize, - imageProviderNotifier: oldItem.imageProviderNotifier, // keep original Notifier - ).._refCount=oldItem._refCount; // keep number of references + final newItem = CacheItem( + value, + filePath: value.path, + previewHash: previewResult.previewHash, + hash: hash, + fileSize: previewResult.fileSize, + imageProviderNotifier: + oldItem.imageProviderNotifier, // keep original Notifier + ).._refCount = oldItem._refCount; // keep number of references // update original fields _items[id] = newItem; - _items[id].invalidateImageProvider; // invalidate imageProvider so it is newly created when needed + _items[id] + .invalidateImageProvider; // invalidate imageProvider so it is newly created when needed } else { - throw Exception( - 'assetCacheAll.replaceImage: unknown type ${value.runtimeType}'); + throw Exception( + 'assetCacheAll.replaceImage: unknown type ${value.runtimeType}'); } } // return File associated with asset, used to save assets when saving note - File getAssetFile(int id){ + File getAssetFile(int id) { final item = _items[id]; if (item.value is File) { return (item.value as File); @@ -693,7 +1017,6 @@ class AssetCacheAll { } } - // generate random file name String generateRandomFileName([String extension = 'txt']) { final timestamp = DateTime.now().millisecondsSinceEpoch; @@ -717,8 +1040,10 @@ class AssetCacheAll { final fileSize = file.lengthSync(); const trailerReadSize = 4096; - final trailerStart = fileSize > trailerReadSize ? fileSize - trailerReadSize : 0; - final trailerLength = fileSize > trailerReadSize ? trailerReadSize : fileSize; + final trailerStart = + fileSize > trailerReadSize ? fileSize - trailerReadSize : 0; + final trailerLength = + fileSize > trailerReadSize ? trailerReadSize : fileSize; final raf = file.openSync(); final trailerBytes = Uint8List(trailerLength); @@ -734,7 +1059,8 @@ class AssetCacheAll { } final trailerSlice = trailerContent.substring(trailerIndex); - final infoMatch = RegExp(r'/Info\s+(\d+)\s+(\d+)\s+R').firstMatch(trailerSlice); + final infoMatch = + RegExp(r'/Info\s+(\d+)\s+(\d+)\s+R').firstMatch(trailerSlice); if (infoMatch == null) { log.info('Info object not found in trailer'); return ''; @@ -752,7 +1078,10 @@ class AssetCacheAll { return ''; } - final startxrefLine = trailerContent.substring(startxrefIndex).split(RegExp(r'\r?\n'))[1].trim(); + final startxrefLine = trailerContent + .substring(startxrefIndex) + .split(RegExp(r'\r?\n'))[1] + .trim(); final xrefOffset = int.tryParse(startxrefLine); if (xrefOffset == null) { log.info('Invalid startxref value'); @@ -764,7 +1093,8 @@ class AssetCacheAll { raf.setPositionSync(xrefOffset); final xrefBuffer = Uint8List(8192); // read enough bytes for xref table final bytesRead = raf.readIntoSync(xrefBuffer); - final xrefContent = ascii.decode(xrefBuffer.sublist(0, bytesRead), allowInvalid: true); + final xrefContent = + ascii.decode(xrefBuffer.sublist(0, bytesRead), allowInvalid: true); // Find object offset in xref table //final objPattern = RegExp(r'(\d{10})\s+(\d{5})\s+n'); @@ -772,7 +1102,8 @@ class AssetCacheAll { int? objOffset; //nt currentObjNumber = 0; - final subMatches = RegExp(r'(\d+)\s+(\d+)\s+(\d+)\s+n').allMatches(xrefContent); + final subMatches = + RegExp(r'(\d+)\s+(\d+)\s+(\d+)\s+n').allMatches(xrefContent); for (final m in subMatches) { final objNum = int.parse(m.group(1)!); final offset = int.parse(m.group(2)!); @@ -792,9 +1123,11 @@ class AssetCacheAll { raf.setPositionSync(objOffset); final objBytes = Uint8List(1024); // read enough for object final objRead = raf.readIntoSync(objBytes); - final objContent = ascii.decode(objBytes.sublist(0, objRead), allowInvalid: true); + final objContent = + ascii.decode(objBytes.sublist(0, objRead), allowInvalid: true); - final objMatchContent = RegExp(r'obj(.*?)endobj', dotAll: true).firstMatch(objContent); + final objMatchContent = + RegExp(r'obj(.*?)endobj', dotAll: true).firstMatch(objContent); if (objMatchContent == null) { log.info('Info object content not found'); raf.closeSync(); @@ -806,12 +1139,12 @@ class AssetCacheAll { final titleMatch = RegExp(r'/Title\s+\((.*?)\)').firstMatch(infoContent); final authorMatch = RegExp(r'/Author\s+\((.*?)\)').firstMatch(infoContent); final creationMatch = - RegExp(r'/CreationDate\s+\((.*?)\)').firstMatch(infoContent); - final String metadata='${titleMatch?.group(1) ?? 'N/A'}${authorMatch?.group(1) ?? 'N/A'}${creationMatch?.group(1) ?? 'N/A'}'; + RegExp(r'/CreationDate\s+\((.*?)\)').firstMatch(infoContent); + final String metadata = + '${titleMatch?.group(1) ?? 'N/A'}${authorMatch?.group(1) ?? 'N/A'}${creationMatch?.group(1) ?? 'N/A'}'; return metadata; } @override String toString() => _items.toString(); - } From 7befb1058e217a77e14c8550c6b5deedc1e37f34 Mon Sep 17 00:00:00 2001 From: QubaB Date: Mon, 6 Oct 2025 08:17:36 +0200 Subject: [PATCH 11/35] assetcacheAll - frmoved all non File cache items. cache now works --- lib/components/canvas/_asset_cache.dart | 313 ++++-------------------- lib/pages/editor/editor.dart | 1 - 2 files changed, 45 insertions(+), 269 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index 69e1c61302..99ad5b66ed 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -9,7 +9,6 @@ import 'package:logging/logging.dart'; import 'package:path_provider/path_provider.dart'; import 'package:pdfrx/pdfrx.dart'; import 'package:saber/components/canvas/image/editor_image.dart'; -import 'package:saber/data/file_manager/file_manager.dart'; class PdfInfoExtractor { /// Extracts /Info dictionary from a PDF file with minimal reads (synchronous) @@ -25,7 +24,6 @@ class PdfInfoExtractor { final isLinearized = startContent.contains('/Linearized'); - String trailerContent; int xrefOffset = 0; // offset of xref on the start of file int xrefOffsetEnd = 0; // offset of xref on the end of file int infoOffset=0; @@ -77,7 +75,7 @@ class PdfInfoExtractor { } if (infoOffset==-1 && xrefOffsetEnd != 0) { // info object is not in this xref table it is at the end of file at xrefOffsetEnd found in /Prev - infoOffset = _findObjectOffset(raf, xrefOffsetEnd, infoObjNum,fileSize, ""); + infoOffset = _findObjectOffset(raf, xrefOffsetEnd, infoObjNum,fileSize, ''); } if (infoOffset<0) { infoOffset = -2; @@ -111,7 +109,7 @@ class PdfInfoExtractor { } xrefOffset = int.parse(startxrefMatch.group(1)!); infoOffset = - _findObjectOffset(raf, xrefOffset, infoObjNum, fileSize, ""); + _findObjectOffset(raf, xrefOffset, infoObjNum, fileSize, ''); } if (infoOffset != 0) { return _extractInfoFromTrailer(raf, infoOffset, fileSize); @@ -123,19 +121,6 @@ class PdfInfoExtractor { } } - static int _findXrefFromEnd(RandomAccessFile raf, int fileSize) { - final endOffset = fileSize > 1024 ? fileSize - 1024 : 0; - raf.setPositionSync(endOffset); - final endBytes = raf.readSync(1024); - final endContent = latin1.decode(endBytes); - - final startxrefMatch = RegExp(r'startxref\s+(\d+)').firstMatch(endContent); - if (startxrefMatch == null) { - throw Exception('startxref not found'); - } - return int.parse(startxrefMatch.group(1)!); - } - static Map _extractInfoFromTrailer( RandomAccessFile raf, int infoOffset, @@ -173,17 +158,15 @@ class PdfInfoExtractor { final xrefLines = xrefContent.split('\n'); int startNum = 0; int count = 0; - int i0=-1; int iStart=-1; - int iStartTable=0; // find line xref for (int i = 0; i < xrefLines.length; i++) { final line = xrefLines[i].trim(); if (line == 'xref') { iStart=i+1; + break; } - break; } if (iStart<0){ throw('xref section does not contain xref'); @@ -423,7 +406,7 @@ class OrderedAssetCache { /// New approach to cache /// /// handles jpg, png, pdf (not svg yet) -/// for each photo item provides ValueNotifier so the same items have the same ImageProvider +/// for each photo item provides ValueNotifier< ImageProvider? > so the same items have the same ImageProvider /// for each pdf item provides PdfDocument every page of pdf use the same provider /// /// During reading note to Editor are new items added using addSync - which is fast @@ -440,7 +423,7 @@ class OrderedAssetCache { /// /// Cache properties: /// 1. Every cache item is created and treated as File (path). Even picked Photos are first saved as temporary files and then added to chache. -/// 2. Each item provides PdfDocument for pdfs or ValueNotifier for images. It saves memory +/// 2. Each item provides PdfDocument for pdfs or ValueNotifier< ImageProvider? > for images. It saves memory // class returning preview data class PreviewResult { @@ -510,6 +493,13 @@ class CacheItem { } } + if (fileInfo != null && other.fileInfo != null) { + if (fileInfo == other.fileInfo) { + // both file info are the same. + return true; + } + } + // Quick check using previewHash if (previewHash != null && other.previewHash != null) { if (previewHash != other.previewHash) { @@ -662,21 +652,22 @@ class AssetCacheAll { // read either the whole file if small, or just the first 100 KB final toRead = fileSize < 100 * 1024 ? fileSize : 100 * 1024; final bytes = raf.readSync(toRead); - final previewHash = calculateHash(bytes); - final firstBytes =ascii.decode(bytes.sublist(0, 4)); // first 4 characters - used to detect PDF file - String fileInfo=""; - if (firstBytes == "%PDF") { + final previewHashBytes = calculateHash(bytes); + final firstBytes =latin1.decode(bytes.sublist(0, 4)); // first 4 characters - used to detect PDF file + String fileInfo=''; + if (firstBytes == '%PDF') { + // asset is pdf, get pdf /Info, it is quick try { final info = PdfInfoExtractor.extractInfo(file); - fileInfo=info.values.join(','); // convert file info to String + fileInfo=info.values.join(','); // convert info to one String } catch(e) { - fileInfo=""; + fileInfo=''; } } return PreviewResult( - (fileSize.hashCode << 32) ^ previewHash, // hash + (fileSize.hashCode << 32) ^ previewHashBytes, // previehash is put together from file size and hash of first 100kB fileSize, // file size firstBytes, // first 4 bytes - used to recognize pdf file format fileInfo @@ -686,11 +677,14 @@ class AssetCacheAll { } } - // add to cache but read only small part of files - used when reading note from disk - // full hashes are established later - // is used during read of note when it is opened. + // Is used during read of note when it is opened. // everything from new notes sba2 is File! // only old notes provide bytes instead File + // NOTE: Because old asset cache stores pdf as asset for each page, we should + // check during reading if asset is not the same as already stored asset! + // It is why we simply not add every asset, because they can be the same. + // add to cache but read only small part of files - used when reading note from disk + // full hashes are established later int addSync(Object value) { if (value is File) { log.info('allCache.addSync: value = $value'); @@ -698,7 +692,7 @@ class AssetCacheAll { final existingPathIndex = _items.indexWhere((i) => i.filePath == path); if (existingPathIndex != -1) { - // file path already in cache so file + // file path already in cache and is the same _items[existingPathIndex].addUse(); log.info( 'allCache.addSync: already in cache {$_items[existingPathIndex]._refCount}'); @@ -709,7 +703,7 @@ class AssetCacheAll { getFilePreviewHash(value); // calculate preliminary hash of file or get Info of pdf file if (previewResult.fileInfo.isNotEmpty){ - // we know information about file so test it + // we know information about file (it is pdf) so test it final existingFileInfoIndex = _items.indexWhere((i) => i.fileInfo == previewResult.fileInfo); if (existingFileInfoIndex != -1) { // file with this fileInfo already in cache @@ -727,6 +721,7 @@ class AssetCacheAll { return existingIndex; } + // create item final newItem = CacheItem(value, filePath: value.path, previewHash: previewResult.previewHash, @@ -739,72 +734,13 @@ class AssetCacheAll { _previewHashIndex[previewResult.previewHash] = index; // add to previously hashed return index; - } else if (value is FileImage) { - final path = value.file.path; - final File file = File(path); - final previewResult = - getFilePreviewHash(file); // calculate preliminary hash of file - - final existingPathIndex = _items.indexWhere((i) => i.filePath == path); - if (existingPathIndex != -1) { - _items[existingPathIndex].addUse(); - return existingPathIndex; - } - - // Check if already cached - if (_previewHashIndex.containsKey(previewResult.previewHash)) { - final existingIndex = _previewHashIndex[previewResult.previewHash]!; - _items[existingIndex].addUse(); - return existingIndex; - } - - final newItem = CacheItem(value, - filePath: path, - previewHash: previewResult.previewHash, - fileSize: previewResult.fileSize) - ..addUse(); - _items.add(newItem); - final index = _items.length - 1; - _previewHashIndex[previewResult.previewHash] = - index; // add to previously hashed - return index; - } else if (value is MemoryImage) { - // file images are first compared by file path - final hash = calculateHash(value.bytes); - final newItem = CacheItem(value, hash: hash)..addUse(); - - final existingHashIndex = _items.indexOf(newItem); - if (existingHashIndex != -1) return existingHashIndex; - - _items.add(newItem); - final index = _items.length - 1; - return index; - } else if (value is List) { - // bytes - final hash = calculateHash(value); - final newItem = CacheItem(value, hash: hash)..addUse(); - - final existingHashIndex = _items.indexOf(newItem); - if (existingHashIndex != -1) return existingHashIndex; - - _items.add(newItem); - final index = _items.length - 1; - return index; - } else if (value is String) { - // directly calculate hash - final newItem = CacheItem(value, hash: value.hashCode)..addUse(); - final existingHashIndex = _items.indexOf(newItem); - if (existingHashIndex != -1) return existingHashIndex; - _items.add(newItem); - final index = _items.length - 1; - return index; - } else { - throw Exception( - 'OrderedAssetCache.getBytes: unknown type ${value.runtimeType}'); + } + else{ + throw Exception('assetCacheAll.add: unknown type ${value.runtimeType}'); } } -// is used from Editor, when adding asset using file picker +// async add cache is used from Editor, when adding asset using file picker // always is used File! Future add(Object value) async { if (value is File) { @@ -817,11 +753,12 @@ class AssetCacheAll { _items[existingPathIndex].addUse(); return existingPathIndex; } + final previewResult = getFilePreviewHash(value); // calculate preliminary hash of file if (previewResult.fileInfo.isNotEmpty){ - // we know information about file so test it + // we know information about file (pdf file) so test it final existingFileInfoIndex = _items.indexWhere((i) => i.fileInfo == previewResult.fileInfo); if (existingFileInfoIndex != -1) { // file with this fileInfo already in cache @@ -832,24 +769,27 @@ class AssetCacheAll { } } - // Check if already cached + // Check previwHash value if (_previewHashIndex.containsKey(previewResult.previewHash)) { final existingIndex = _previewHashIndex[previewResult.previewHash]!; _items[existingIndex].addUse(); return existingIndex; } - // compute expensive content hash + // compute expensive content hash - need to read whole file final bytes = await value.readAsBytes(); final hash = calculateHash(bytes); + // prepare cache item final newItem = CacheItem(value, filePath: value.path, previewHash: previewResult.previewHash, hash: hash, - fileSize: previewResult.fileSize) + fileSize: previewResult.fileSize, + fileInfo: previewResult.fileInfo) ..addUse(); + // check if it is already in cache final existingHashIndex = _items.indexOf(newItem); if (existingHashIndex != -1) { _items[existingHashIndex].addUse(); @@ -860,58 +800,9 @@ class AssetCacheAll { _previewHashIndex[previewResult.previewHash] = index; // add to previously hashed return index; - } else if (value is FileImage) { - // file images are first compared by file path - final path = value.file.path; - - // 1. Fast path check - final existingPathIndex = _items.indexWhere((i) => i.filePath == path); - if (existingPathIndex != -1) { - _items[existingPathIndex].addUse(); - return existingPathIndex; - } - - final File file = File(path); - final previewResult = - getFilePreviewHash(file); // calculate preliminary hash of file - - // Check if already cached - if (_previewHashIndex.containsKey(previewResult.previewHash)) { - final existingIndex = _previewHashIndex[previewResult.previewHash]!; - _items[existingIndex].addUse(); - return existingIndex; - } - - final bytes = await value.file.readAsBytes(); - final hash = calculateHash(bytes); - - final newItem = CacheItem(value, hash: hash, filePath: path); - - final existingHashIndex = _items.indexOf(newItem); - if (existingHashIndex != -1) return existingHashIndex; - - _items.add(newItem); - return _items.length - 1; - } else if (value is MemoryImage) { - // file images are first compared by file path - final hash = calculateHash(value.bytes); - - final newItem = CacheItem(value, hash: hash); - - final existingHashIndex = _items.indexOf(newItem); - if (existingHashIndex != -1) return existingHashIndex; - - _items.add(newItem); - return _items.length - 1; - } else { - final hash = value.hashCode; // string - final newItem = CacheItem(value, hash: hash); - - final existingIndex = _items.indexOf(newItem); - if (existingIndex != -1) return existingIndex; - - _items.add(newItem); - return _items.length - 1; + } else{ + throw Exception( + 'assetCacheAll.add: unknown type ${value.runtimeType}'); } } @@ -968,7 +859,7 @@ class AssetCacheAll { } } - /// retunr real index through alias map + /// return real index through alias map int resolveIndex(int index) { return _aliasMap[index] ?? index; } @@ -1031,120 +922,6 @@ class AssetCacheAll { return await file.writeAsBytes(bytes, flush: true); } - String readPdfMetadataSync(File file) { - if (!file.existsSync()) { - log.info('File not found'); - return ''; - } - - final fileSize = file.lengthSync(); - const trailerReadSize = 4096; - - final trailerStart = - fileSize > trailerReadSize ? fileSize - trailerReadSize : 0; - final trailerLength = - fileSize > trailerReadSize ? trailerReadSize : fileSize; - - final raf = file.openSync(); - final trailerBytes = Uint8List(trailerLength); - raf.setPositionSync(trailerStart); - raf.readIntoSync(trailerBytes); - raf.closeSync(); - - final trailerContent = ascii.decode(trailerBytes, allowInvalid: true); - final trailerIndex = trailerContent.lastIndexOf('trailer'); - if (trailerIndex == -1) { - log.info('Trailer not found'); - return ''; - } - - final trailerSlice = trailerContent.substring(trailerIndex); - final infoMatch = - RegExp(r'/Info\s+(\d+)\s+(\d+)\s+R').firstMatch(trailerSlice); - if (infoMatch == null) { - log.info('Info object not found in trailer'); - return ''; - } - - final infoObjNumber = int.parse(infoMatch.group(1)!); - final infoObjGen = int.parse(infoMatch.group(2)!); - log.info('Info object reference: $infoObjNumber $infoObjGen'); - - // Find startxref - final startxrefIndex = trailerContent.lastIndexOf('startxref'); - if (startxrefIndex == -1) { - log.info('startxref not found'); - raf.closeSync(); - return ''; - } - - final startxrefLine = trailerContent - .substring(startxrefIndex) - .split(RegExp(r'\r?\n'))[1] - .trim(); - final xrefOffset = int.tryParse(startxrefLine); - if (xrefOffset == null) { - log.info('Invalid startxref value'); - raf.closeSync(); - return ''; - } - - // Go to xref table - raf.setPositionSync(xrefOffset); - final xrefBuffer = Uint8List(8192); // read enough bytes for xref table - final bytesRead = raf.readIntoSync(xrefBuffer); - final xrefContent = - ascii.decode(xrefBuffer.sublist(0, bytesRead), allowInvalid: true); - - // Find object offset in xref table - //final objPattern = RegExp(r'(\d{10})\s+(\d{5})\s+n'); - //final objMatches = objPattern.allMatches(xrefContent); - int? objOffset; - - //nt currentObjNumber = 0; - final subMatches = - RegExp(r'(\d+)\s+(\d+)\s+(\d+)\s+n').allMatches(xrefContent); - for (final m in subMatches) { - final objNum = int.parse(m.group(1)!); - final offset = int.parse(m.group(2)!); - if (objNum == infoObjNumber) { - objOffset = offset; - break; - } - } - - if (objOffset == null) { - log.info('Info object offset not found in xref table'); - raf.closeSync(); - return ''; - } - - // Read Info object directly - raf.setPositionSync(objOffset); - final objBytes = Uint8List(1024); // read enough for object - final objRead = raf.readIntoSync(objBytes); - final objContent = - ascii.decode(objBytes.sublist(0, objRead), allowInvalid: true); - - final objMatchContent = - RegExp(r'obj(.*?)endobj', dotAll: true).firstMatch(objContent); - if (objMatchContent == null) { - log.info('Info object content not found'); - raf.closeSync(); - return ''; - } - - final infoContent = objMatchContent.group(1)!; - - final titleMatch = RegExp(r'/Title\s+\((.*?)\)').firstMatch(infoContent); - final authorMatch = RegExp(r'/Author\s+\((.*?)\)').firstMatch(infoContent); - final creationMatch = - RegExp(r'/CreationDate\s+\((.*?)\)').firstMatch(infoContent); - final String metadata = - '${titleMatch?.group(1) ?? 'N/A'}${authorMatch?.group(1) ?? 'N/A'}${creationMatch?.group(1) ?? 'N/A'}'; - return metadata; - } - @override String toString() => _items.toString(); } diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index 1ae47759bb..edfd16ce3a 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -12,7 +12,6 @@ import 'package:flutter/services.dart'; import 'package:flutter_quill/flutter_quill.dart' as flutter_quill; import 'package:keybinder/keybinder.dart'; import 'package:logging/logging.dart'; -import 'package:path_provider/path_provider.dart'; import 'package:printing/printing.dart'; import 'package:saber/components/canvas/_asset_cache.dart'; import 'package:saber/components/canvas/_stroke.dart'; From 7b6876c49f0f199c692f3b00e01d0b5181e5b986 Mon Sep 17 00:00:00 2001 From: QubaB Date: Mon, 6 Oct 2025 11:36:41 +0200 Subject: [PATCH 12/35] set correct asset id during saving --- lib/components/canvas/image/pdf_editor_image.dart | 2 +- lib/components/canvas/image/png_editor_image.dart | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/components/canvas/image/pdf_editor_image.dart b/lib/components/canvas/image/pdf_editor_image.dart index 6af498134c..1fa2586f77 100644 --- a/lib/components/canvas/image/pdf_editor_image.dart +++ b/lib/components/canvas/image/pdf_editor_image.dart @@ -132,7 +132,7 @@ class PdfEditorImage extends EditorImage { assert(!json.containsKey('a')); assert(!json.containsKey('b')); - json['a'] = assets.add(pdfFile ?? pdfBytes!); + json['a'] = assetId ;// assets.add(pdfFile ?? pdfBytes!); json['pdfi'] = pdfPage; return json; diff --git a/lib/components/canvas/image/png_editor_image.dart b/lib/components/canvas/image/png_editor_image.dart index 812bfaecdd..f64170f553 100644 --- a/lib/components/canvas/image/png_editor_image.dart +++ b/lib/components/canvas/image/png_editor_image.dart @@ -143,7 +143,7 @@ class PngEditorImage extends EditorImage { @override Map toJson(OrderedAssetCache assets) => super.toJson(assets) ..addAll({ - if (imageProvider != null) 'a': assets.add(imageProvider!), + if (imageProvider != null) 'a': assetId, }); @override From d4b5aaa196ee79b8440d7d947f98c1003a54cc99 Mon Sep 17 00:00:00 2001 From: QubaB Date: Mon, 6 Oct 2025 11:37:15 +0200 Subject: [PATCH 13/35] fileManager - skip copy file to itself (it created 0B file) --- lib/data/file_manager/file_manager.dart | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/data/file_manager/file_manager.dart b/lib/data/file_manager/file_manager.dart index 053884265e..ffb1c56d7f 100644 --- a/lib/data/file_manager/file_manager.dart +++ b/lib/data/file_manager/file_manager.dart @@ -270,6 +270,10 @@ class FileManager { await _createFileDirectory(filePath); // create directory filePath is "relative to saber documents directory") filePath = getFilePath(filePath); // if needed add documents directory to file path to have full path + if (fileFrom.path == filePath){ + // file is copied to itself, do nothing (it happens when asset is saved with the same name) + return; + } log.fine('Copying to $filePath'); await _saveFileAsRecentlyAccessed(filePath); From 69127c2e56929941dd5d7465a917890353c50184 Mon Sep 17 00:00:00 2001 From: QubaB Date: Mon, 6 Oct 2025 14:54:27 +0200 Subject: [PATCH 14/35] removed assetsCache and implemented only new cache svg files are not handled --- lib/components/canvas/_asset_cache.dart | 14 +++++-- .../canvas/canvas_image_dialog.dart | 12 +----- lib/components/canvas/image/editor_image.dart | 8 +--- .../canvas/image/pdf_editor_image.dart | 34 ++++++----------- .../canvas/image/png_editor_image.dart | 6 +-- .../canvas/image/svg_editor_image.dart | 37 +++++++++++-------- lib/data/editor/editor_core_info.dart | 37 ++++++------------- lib/data/editor/page.dart | 13 ++----- lib/pages/editor/editor.dart | 10 +---- 9 files changed, 63 insertions(+), 108 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index 99ad5b66ed..b53c96e4f2 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -443,6 +443,7 @@ class CacheItem { final ValueNotifier imageProviderNotifier; // image provider for png, svg as value listener PdfDocument? _pdfDocument; // pdf document provider for pdf + Uint8List? bytes; // used only when reading inline assets // for files only final int? fileSize; @@ -460,6 +461,7 @@ class CacheItem { this.fileSize, this.fileExt, this.fileInfo, + this.bytes, ValueNotifier? imageProviderNotifier, }) : imageProviderNotifier = imageProviderNotifier ?? ValueNotifier(null); @@ -916,10 +918,16 @@ class AssetCacheAll { } // create temporary file from bytes when inline bytes are read - Future createRuntimeFile(String ext, Uint8List bytes) async { - final dir = await getApplicationSupportDirectory(); + File createRuntimeFile(String ext, Uint8List bytes) { + final dir = Directory.systemTemp; // Použití systémového temp adresáře +// final dir = await getApplicationSupportDirectory(); final file = File('${dir.path}/TmPmP_${generateRandomFileName(ext)}'); - return await file.writeAsBytes(bytes, flush: true); + file.writeAsBytesSync(bytes, flush: true); + return file; + } + + void dispose() { + _items.clear(); } @override diff --git a/lib/components/canvas/canvas_image_dialog.dart b/lib/components/canvas/canvas_image_dialog.dart index 7881188076..23aa1d514d 100644 --- a/lib/components/canvas/canvas_image_dialog.dart +++ b/lib/components/canvas/canvas_image_dialog.dart @@ -70,7 +70,7 @@ class _CanvasImageDialogState extends State { switch (widget.image) { case PdfEditorImage image: if (!image.loadedIn) await image.loadIn(); - bytes = image.pdfBytes!; + bytes = await image.assetCacheAll.getBytes(image.assetId); case SvgEditorImage image: bytes = switch (image.svgLoader) { (SvgStringLoader loader) => @@ -80,15 +80,7 @@ class _CanvasImageDialogState extends State { image.svgLoader, 'svgLoader', 'Unknown SVG loader type'), }; case PngEditorImage image: - if (image.imageProvider is MemoryImage) { - bytes = (image.imageProvider as MemoryImage).bytes; - } else if (image.imageProvider is FileImage) { - bytes = - await (image.imageProvider as FileImage).file.readAsBytes(); - } else { - throw ArgumentError.value(image.imageProvider, 'imageProvider', - 'Unknown image provider type'); - } + bytes = await image.assetCacheAll.getBytes(image.assetId); } if (!context.mounted) return; FileManager.exportFile(imageFileName, bytes, diff --git a/lib/components/canvas/image/editor_image.dart b/lib/components/canvas/image/editor_image.dart index a7afd68494..63f070862c 100644 --- a/lib/components/canvas/image/editor_image.dart +++ b/lib/components/canvas/image/editor_image.dart @@ -33,7 +33,6 @@ sealed class EditorImage extends ChangeNotifier { /// This is used when "downloading" the image to the user's photo gallery. final String extension; - final AssetCache assetCache; final AssetCacheAll assetCacheAll; bool _isThumbnail = false; @@ -89,7 +88,6 @@ sealed class EditorImage extends ChangeNotifier { @protected EditorImage({ required this.id, - required this.assetCache, required this.assetCacheAll, required this.extension, required this.pageIndex, @@ -114,7 +112,6 @@ sealed class EditorImage extends ChangeNotifier { required List? inlineAssets, bool isThumbnail = false, required String sbnPath, - required AssetCache assetCache, required AssetCacheAll assetCacheAll, }) { String? extension = json['e']; @@ -124,7 +121,6 @@ sealed class EditorImage extends ChangeNotifier { inlineAssets: inlineAssets, isThumbnail: isThumbnail, sbnPath: sbnPath, - assetCache: assetCache, assetCacheAll: assetCacheAll, ); } else if (extension == '.pdf') { @@ -133,7 +129,6 @@ sealed class EditorImage extends ChangeNotifier { inlineAssets: inlineAssets, isThumbnail: isThumbnail, sbnPath: sbnPath, - assetCache: assetCache, assetCacheAll: assetCacheAll, ); } else { @@ -142,7 +137,6 @@ sealed class EditorImage extends ChangeNotifier { inlineAssets: inlineAssets, isThumbnail: isThumbnail, sbnPath: sbnPath, - assetCache: assetCache, assetCacheAll: assetCacheAll, ); } @@ -150,7 +144,7 @@ sealed class EditorImage extends ChangeNotifier { @mustBeOverridden @mustCallSuper - Map toJson(OrderedAssetCache assets) => { + Map toJson() => { 'id': id, 'e': extension, 'i': pageIndex, diff --git a/lib/components/canvas/image/pdf_editor_image.dart b/lib/components/canvas/image/pdf_editor_image.dart index 1fa2586f77..6640304fa5 100644 --- a/lib/components/canvas/image/pdf_editor_image.dart +++ b/lib/components/canvas/image/pdf_editor_image.dart @@ -4,7 +4,6 @@ class PdfEditorImage extends EditorImage { /// index of asset assigned to this pdf file int assetId; - Uint8List? pdfBytes; final int pdfPage; /// If the pdf needs to be loaded from disk, this is the File @@ -17,10 +16,8 @@ class PdfEditorImage extends EditorImage { PdfEditorImage({ required super.id, - required super.assetCache, required super.assetCacheAll, required this.assetId, - required this.pdfBytes, required this.pdfFile, required this.pdfPage, required super.pageIndex, @@ -37,8 +34,8 @@ class PdfEditorImage extends EditorImage { super.isThumbnail, }) : assert( !naturalSize.isEmpty, 'naturalSize must be set for PdfEditorImage'), - assert(pdfBytes != null || pdfFile != null, - 'pdfFile must be set if pdfBytes is null'), + assert(pdfFile != null, + 'pdfFile must be set'), super( extension: '.pdf', srcRect: Rect.zero, @@ -49,7 +46,6 @@ class PdfEditorImage extends EditorImage { required List? inlineAssets, bool isThumbnail = false, required String sbnPath, - required AssetCache assetCache, required AssetCacheAll assetCacheAll, }) { String? extension = json['e'] as String?; @@ -63,27 +59,25 @@ class PdfEditorImage extends EditorImage { if (inlineAssets == null) { pdfFile = FileManager.getFile('$sbnPath${Editor.extension}.$assetIndexJson'); - pdfBytes = assetCache.get(pdfFile); + assetIndex = assetCacheAll.addSync(pdfFile); } else { pdfBytes = inlineAssets[assetIndexJson]; + final tempFile=assetCacheAll.createRuntimeFile('.pdf',pdfBytes); // store to file + assetIndex = assetCacheAll.addSync(tempFile); } } else { if (kDebugMode) { throw Exception('PdfEditorImage.fromJson: pdf bytes not found'); } pdfBytes = Uint8List(0); + final tempFile=assetCacheAll.createRuntimeFile('.pdf',pdfBytes); + assetIndex = assetCacheAll.addSync(tempFile); } - assert(pdfBytes != null || pdfFile != null, + assert(assetIndex >0, 'Either pdfBytes or pdfFile must be non-null'); // add to asset cache - if (pdfFile != null) { - assetIndex = assetCacheAll.addSync(pdfFile); - } - else { - assetIndex = assetCacheAll.addSync(pdfBytes!); - } if (assetIndex<0){ throw Exception('EditorImage.fromJson: pdf image not in assets'); } @@ -91,10 +85,8 @@ class PdfEditorImage extends EditorImage { return PdfEditorImage( id: json['id'] ?? -1, // -1 will be replaced by EditorCoreInfo._handleEmptyImageIds() - assetCache: assetCache, assetCacheAll: assetCacheAll, assetId: assetIndex, - pdfBytes: pdfBytes, pdfFile: pdfFile, pdfPage: json['pdfi'], pageIndex: json['i'] ?? 0, @@ -122,17 +114,15 @@ class PdfEditorImage extends EditorImage { } @override - Map toJson(OrderedAssetCache assets) { - final json = super.toJson( - assets, - ); + Map toJson() { + final json = super.toJson(); // remove non-pdf fields json.remove('t'); // thumbnail bytes assert(!json.containsKey('a')); assert(!json.containsKey('b')); - json['a'] = assetId ;// assets.add(pdfFile ?? pdfBytes!); + json['a'] = assetId; json['pdfi'] = pdfPage; return json; @@ -207,10 +197,8 @@ class PdfEditorImage extends EditorImage { @override PdfEditorImage copy() => PdfEditorImage( id: id, - assetCache: assetCache, assetCacheAll: assetCacheAll, assetId: assetId, - pdfBytes: pdfBytes, pdfPage: pdfPage, pdfFile: pdfFile, pageIndex: pageIndex, diff --git a/lib/components/canvas/image/png_editor_image.dart b/lib/components/canvas/image/png_editor_image.dart index f64170f553..c96c306201 100644 --- a/lib/components/canvas/image/png_editor_image.dart +++ b/lib/components/canvas/image/png_editor_image.dart @@ -33,7 +33,6 @@ class PngEditorImage extends EditorImage { PngEditorImage({ required super.id, - required super.assetCache, required super.assetCacheAll, required this.assetId, required super.extension, @@ -60,7 +59,6 @@ class PngEditorImage extends EditorImage { required List? inlineAssets, bool isThumbnail = false, required String sbnPath, - required AssetCache assetCache, required AssetCacheAll assetCacheAll, }) { final assetIndexJson = json['a'] as int?; @@ -102,7 +100,6 @@ class PngEditorImage extends EditorImage { return PngEditorImage( // -1 will be replaced by [EditorCoreInfo._handleEmptyImageIds()] id: json['id'] ?? -1, - assetCache: assetCache, assetCacheAll: assetCacheAll, assetId: assetIndex, extension: json['e'] ?? '.jpg', @@ -141,7 +138,7 @@ class PngEditorImage extends EditorImage { } @override - Map toJson(OrderedAssetCache assets) => super.toJson(assets) + Map toJson() => super.toJson() ..addAll({ if (imageProvider != null) 'a': assetId, }); @@ -260,7 +257,6 @@ class PngEditorImage extends EditorImage { @override PngEditorImage copy() => PngEditorImage( id: id, - assetCache: assetCache, assetCacheAll: assetCacheAll, assetId: assetId, extension: extension, diff --git a/lib/components/canvas/image/svg_editor_image.dart b/lib/components/canvas/image/svg_editor_image.dart index b74417d8c0..d5da3b95b5 100644 --- a/lib/components/canvas/image/svg_editor_image.dart +++ b/lib/components/canvas/image/svg_editor_image.dart @@ -2,17 +2,14 @@ part of 'editor_image.dart'; class SvgEditorImage extends EditorImage { late SvgLoader svgLoader; + int assetId; static final log = Logger('SvgEditorImage'); - @override - @Deprecated('Use the file directly instead') - AssetCache get assetCache => super.assetCache; - SvgEditorImage({ required super.id, - required super.assetCache, required super.assetCacheAll, + required this.assetId, required String? svgString, required File? svgFile, required super.pageIndex, @@ -45,22 +42,21 @@ class SvgEditorImage extends EditorImage { required List? inlineAssets, bool isThumbnail = false, required String sbnPath, - required AssetCache assetCache, required AssetCacheAll assetCacheAll, }) { String? extension = json['e'] as String?; assert(extension == null || extension == '.svg'); - final assetIndex = json['a'] as int?; + final assetIndexJson = json['a'] as int?; + final int? assetIndex; final String? svgString; File? svgFile; - if (assetIndex != null) { + if (assetIndexJson != null) { if (inlineAssets == null) { svgFile = - FileManager.getFile('$sbnPath${Editor.extension}.$assetIndex'); - svgString = assetCache.get(svgFile); + FileManager.getFile('$sbnPath${Editor.extension}.$assetIndexJson'); } else { - svgString = utf8.decode(inlineAssets[assetIndex]); + svgString = utf8.decode(inlineAssets[assetIndexJson]); } } else if (json['b'] != null) { svgString = json['b'] as String; @@ -68,12 +64,21 @@ class SvgEditorImage extends EditorImage { log.warning('SvgEditorImage.fromJson: no svg string found'); svgString = ''; } + if (svgFile != null) { + assetIndex = assetCacheAll.addSync(svgFile); + } + else { + throw Exception('EditorImage.fromJson: svg image not in assets'); + } + if (assetIndex<0){ + throw Exception('EditorImage.fromJson: svg image not in assets'); + } return SvgEditorImage( id: json['id'] ?? -1, // -1 will be replaced by EditorCoreInfo._handleEmptyImageIds() - assetCache: assetCache, assetCacheAll: assetCacheAll, + assetId: assetIndex, svgString: svgString, svgFile: svgFile, pageIndex: json['i'] ?? 0, @@ -107,8 +112,8 @@ class SvgEditorImage extends EditorImage { } @override - Map toJson(OrderedAssetCache assets) { - final json = super.toJson(assets); + Map toJson() { + final json = super.toJson(); // remove non-svg fields json.remove('t'); // thumbnail bytes @@ -116,7 +121,7 @@ class SvgEditorImage extends EditorImage { assert(!json.containsKey('b')); final svgData = _extractSvg(); - json['a'] = assets.add(svgData.string ?? svgData.file!); + json['a'] = assetId; return json; } @@ -197,8 +202,8 @@ class SvgEditorImage extends EditorImage { return SvgEditorImage( id: id, // ignore: deprecated_member_use_from_same_package - assetCache: assetCache, assetCacheAll: assetCacheAll, + assetId: assetId, svgString: svgData.string, svgFile: svgData.file, pageIndex: pageIndex, diff --git a/lib/data/editor/editor_core_info.dart b/lib/data/editor/editor_core_info.dart index 8ae5d01ff7..11c39137f8 100644 --- a/lib/data/editor/editor_core_info.dart +++ b/lib/data/editor/editor_core_info.dart @@ -57,7 +57,6 @@ class EditorCoreInfo { /// The file name without its parent directories. String get fileName => filePath.substring(filePath.lastIndexOf('/') + 1); - AssetCache assetCache; AssetCacheAll assetCacheAll; int nextImageId; Color? backgroundColor; @@ -80,7 +79,6 @@ class EditorCoreInfo { lineThickness: stows.lastLineThickness.value, pages: [], initialPageIndex: null, - assetCache: null, assetCacheAll: null, ).._migrateOldStrokesAndImages( fileVersion: sbnVersion, @@ -102,7 +100,6 @@ class EditorCoreInfo { lineHeight = stows.lastLineHeight.value, lineThickness = stows.lastLineThickness.value, pages = [], - assetCache = AssetCache(), assetCacheAll = AssetCacheAll(); EditorCoreInfo._({ @@ -116,10 +113,8 @@ class EditorCoreInfo { required this.lineThickness, required this.pages, required this.initialPageIndex, - required AssetCache? assetCache, required AssetCacheAll? assetCacheAll, - }) : assetCache = assetCache ?? AssetCache(), - assetCacheAll = assetCacheAll ?? AssetCacheAll() + }) : assetCacheAll = assetCacheAll ?? AssetCacheAll() { _handleEmptyImageIds(); } @@ -162,7 +157,6 @@ class EditorCoreInfo { 'Invalid color value: (${json['b'].runtimeType}) ${json['b']}'); } - final assetCache = AssetCache(); final assetCacheAll = AssetCacheAll(); return EditorCoreInfo._( @@ -187,11 +181,9 @@ class EditorCoreInfo { onlyFirstPage: onlyFirstPage, fileVersion: fileVersion, sbnPath: filePath, - assetCache: assetCache, assetCacheAll: assetCacheAll, ), initialPageIndex: json['c'] as int?, - assetCache: assetCache, assetCacheAll: assetCacheAll, ) .._migrateOldStrokesAndImages( @@ -218,7 +210,6 @@ class EditorCoreInfo { lineHeight = stows.lastLineHeight.value, lineThickness = stows.lastLineThickness.value, pages = [], - assetCache = AssetCache(), assetCacheAll = AssetCacheAll(){ _migrateOldStrokesAndImages( fileVersion: 0, @@ -237,7 +228,6 @@ class EditorCoreInfo { required bool onlyFirstPage, required int fileVersion, required String sbnPath, - required AssetCache assetCache, required AssetCacheAll assetCacheAll, }) { if (pages == null || pages.isEmpty) return []; @@ -259,7 +249,6 @@ class EditorCoreInfo { readOnly: readOnly, fileVersion: fileVersion, sbnPath: sbnPath, - assetCache: assetCache, assetCacheAll: assetCacheAll, )) .toList(); @@ -317,7 +306,6 @@ class EditorCoreInfo { isThumbnail: readOnly, onlyFirstPage: onlyFirstPage, sbnPath: filePath, - assetCache: assetCache, assetCacheAll: assetCacheAll, ); for (EditorImage image in images) { @@ -478,10 +466,8 @@ class EditorCoreInfo { /// Returns the json map and a list of assets. /// Assets are stored in separate files. - (Map json, OrderedAssetCache) toJson() { + Map toJson() { /// This will be populated in various [toJson] methods. - final OrderedAssetCache assets = OrderedAssetCache(); - final json = { 'v': sbnVersion, 'ni': nextImageId, @@ -489,11 +475,11 @@ class EditorCoreInfo { 'p': backgroundPattern.name, 'l': lineHeight, 'lt': lineThickness, - 'z': pages.map((EditorPage page) => page.toJson(assets)).toList(), + 'z': pages.map((EditorPage page) => page.toJson()).toList(), 'c': initialPageIndex, }; - return (json, assets); + return (json); } /// Converts the current note as an SBA (Saber Archive) file, @@ -508,7 +494,7 @@ class EditorCoreInfo { Future> saveToSba({ required int? currentPageIndex, }) async { - final (bson, assets) = saveToBinary( + final (bson) = saveToBinary( currentPageIndex: currentPageIndex, ); const filePath = 'main${Editor.extension}'; @@ -521,8 +507,8 @@ class EditorCoreInfo { )); await Future.wait([ - for (int i = 0; i < assets.length; ++i) - assets.getBytes(i).then((bytes) => archive.addFile(ArchiveFile( + for (int i = 0; i < assetCacheAll.length; ++i) + assetCacheAll.getBytes(i).then((bytes) => archive.addFile(ArchiveFile( '$filePath.$i', bytes.length, bytes, @@ -533,20 +519,20 @@ class EditorCoreInfo { } /// Returns the bson bytes and the assets. - (Uint8List bson, OrderedAssetCache assets) saveToBinary({ + Uint8List saveToBinary({ required int? currentPageIndex, }) { initialPageIndex = currentPageIndex ?? initialPageIndex; - final (json, assets) = toJson(); + final (json) = toJson(); final bson = BsonCodec.serialize(json); - return (bson.byteList, assets); + return bson.byteList; } void dispose() { for (final page in pages) { page.dispose(); } - assetCache.dispose(); + assetCacheAll.dispose(); } EditorCoreInfo copyWith({ @@ -573,7 +559,6 @@ class EditorCoreInfo { lineThickness: lineThickness ?? this.lineThickness, pages: pages ?? this.pages, initialPageIndex: initialPageIndex, - assetCache: assetCache, assetCacheAll: assetCacheAll, ); } diff --git a/lib/data/editor/page.dart b/lib/data/editor/page.dart index b35d9ee6fa..f86296d188 100644 --- a/lib/data/editor/page.dart +++ b/lib/data/editor/page.dart @@ -130,7 +130,6 @@ class EditorPage extends ChangeNotifier implements HasSize { required bool readOnly, required int fileVersion, required String sbnPath, - required AssetCache assetCache, required AssetCacheAll assetCacheAll, }) { final size = Size(json['w'] ?? defaultWidth, json['h'] ?? defaultHeight); @@ -148,7 +147,6 @@ class EditorPage extends ChangeNotifier implements HasSize { isThumbnail: readOnly, onlyFirstPage: false, sbnPath: sbnPath, - assetCache: assetCache, assetCacheAll: assetCacheAll, ), quill: QuillStruct( @@ -166,23 +164,22 @@ class EditorPage extends ChangeNotifier implements HasSize { inlineAssets: inlineAssets, isThumbnail: false, sbnPath: sbnPath, - assetCache: assetCache, assetCacheAll: assetCacheAll, ) : null, ); } - Map toJson(OrderedAssetCache assets) => { + Map toJson() => { 'w': size.width, 'h': size.height, if (strokes.isNotEmpty) 's': strokes.map((stroke) => stroke.toJson()).toList(), if (images.isNotEmpty) - 'i': images.map((image) => image.toJson(assets)).toList(), + 'i': images.map((image) => image.toJson()).toList(), if (!quill.controller.document.isEmpty()) 'q': quill.controller.document.toDelta().toJson(), - if (backgroundImage != null) 'b': backgroundImage?.toJson(assets) + if (backgroundImage != null) 'b': backgroundImage?.toJson() }; /// Inserts a stroke, while keeping the strokes sorted by @@ -245,7 +242,6 @@ class EditorPage extends ChangeNotifier implements HasSize { required bool isThumbnail, required bool onlyFirstPage, required String sbnPath, - required AssetCache assetCache, required AssetCacheAll assetCacheAll, }) => images @@ -257,7 +253,6 @@ class EditorPage extends ChangeNotifier implements HasSize { inlineAssets: inlineAssets, isThumbnail: isThumbnail, sbnPath: sbnPath, - assetCache: assetCache, assetCacheAll: assetCacheAll, ); }) @@ -271,7 +266,6 @@ class EditorPage extends ChangeNotifier implements HasSize { required List? inlineAssets, required bool isThumbnail, required String sbnPath, - required AssetCache assetCache, required AssetCacheAll assetCacheAll, }) => EditorImage.fromJson( @@ -279,7 +273,6 @@ class EditorPage extends ChangeNotifier implements HasSize { inlineAssets: inlineAssets, isThumbnail: isThumbnail, sbnPath: sbnPath, - assetCache: assetCache, assetCacheAll: assetCacheAll, ); diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index edfd16ce3a..b363f62b8a 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -903,16 +903,13 @@ class EditorState extends State { final filePath = coreInfo.filePath + Editor.extension; final Uint8List bson; - final OrderedAssetCache assets; - coreInfo.assetCache.allowRemovingAssets = false; coreInfo.assetCacheAll.allowRemovingAssets = false; try { // go through all pages and prepare Json of each page. - (bson, assets) = coreInfo.saveToBinary( + (bson) = coreInfo.saveToBinary( currentPageIndex: currentPageIndex, ); } finally { - coreInfo.assetCache.allowRemovingAssets = true; coreInfo.assetCacheAll.allowRemovingAssets = true; } try { @@ -1101,8 +1098,8 @@ class EditorState extends State { onDeleteImage: onDeleteImage, onMiscChange: autosaveAfterDelay, onLoad: () => setState(() {}), - assetCache: coreInfo.assetCache, assetCacheAll: coreInfo.assetCacheAll, + assetId: assetIndex, )); } else { @@ -1118,7 +1115,6 @@ class EditorState extends State { onDeleteImage: onDeleteImage, onMiscChange: autosaveAfterDelay, onLoad: () => setState(() {}), - assetCache: coreInfo.assetCache, assetCacheAll: coreInfo.assetCacheAll, assetId: assetIndex, )); @@ -1230,7 +1226,6 @@ class EditorState extends State { ); page.backgroundImage = PdfEditorImage( id: coreInfo.nextImageId++, - pdfBytes: pdfBytes, pdfFile: pdfFile, pdfPage: currentPdfPage, pageIndex: coreInfo.pages.length, @@ -1240,7 +1235,6 @@ class EditorState extends State { onDeleteImage: onDeleteImage, onMiscChange: autosaveAfterDelay, onLoad: () => setState(() {}), - assetCache: coreInfo.assetCache, assetCacheAll: coreInfo.assetCacheAll, assetId: assetIndex, ); From 08d9920042a4434825a0c72709072e2fc80b9282 Mon Sep 17 00:00:00 2001 From: QubaB Date: Mon, 6 Oct 2025 17:12:10 +0200 Subject: [PATCH 15/35] assetcache addUse a removeUse - used on delete image and undo/redo --- lib/components/canvas/_asset_cache.dart | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index b53c96e4f2..323f8e620e 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -6,7 +6,6 @@ import 'package:collection/collection.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:logging/logging.dart'; -import 'package:path_provider/path_provider.dart'; import 'package:pdfrx/pdfrx.dart'; import 'package:saber/components/canvas/image/editor_image.dart'; @@ -467,14 +466,14 @@ class CacheItem { // increase use of item void addUse() { - if (_released) throw StateError('Trying to add use of released CacheItem'); + //if (_released) throw StateError('Trying to add use of released CacheItem'); _refCount++; } // when asset is released (no more used) void freeUse() { if (_refCount > 0) _refCount--; - if (_refCount == 0) _released = true; + //if (_refCount == 0) _released = true; } bool get isUnused => _refCount == 0; @@ -679,6 +678,16 @@ class AssetCacheAll { } } + // remove item use but keep it in cache (in case of undo/redo) + void removeUse(int id){ + _items[id].freeUse(); + } + + // add item use + void addUse(int id){ + _items[id].addUse(); + } + // Is used during read of note when it is opened. // everything from new notes sba2 is File! // only old notes provide bytes instead File From 13d411180185daf0b8a99b9b601f8a301b390b88 Mon Sep 17 00:00:00 2001 From: QubaB Date: Mon, 6 Oct 2025 17:12:40 +0200 Subject: [PATCH 16/35] svgImage is rewriten to use only File not String --- .../canvas/image/svg_editor_image.dart | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/lib/components/canvas/image/svg_editor_image.dart b/lib/components/canvas/image/svg_editor_image.dart index d5da3b95b5..62d17981de 100644 --- a/lib/components/canvas/image/svg_editor_image.dart +++ b/lib/components/canvas/image/svg_editor_image.dart @@ -10,8 +10,6 @@ class SvgEditorImage extends EditorImage { required super.id, required super.assetCacheAll, required this.assetId, - required String? svgString, - required File? svgFile, required super.pageIndex, required super.pageSize, super.invertible, @@ -25,16 +23,12 @@ class SvgEditorImage extends EditorImage { super.srcRect, super.naturalSize, super.isThumbnail, - }) : assert(svgString != null || svgFile != null, - 'svgFile must be set if svgString is null'), + }) : assert(assetId >-1 , + 'assetId must be set'), super( extension: '.svg', ) { - if (svgString != null) { - svgLoader = SvgStringLoader(svgString); - } else { - svgLoader = SvgFileLoader(svgFile!); - } + svgLoader = SvgFileLoader(assetCacheAll.getAssetFile(assetId)); } factory SvgEditorImage.fromJson( @@ -48,21 +42,22 @@ class SvgEditorImage extends EditorImage { assert(extension == null || extension == '.svg'); final assetIndexJson = json['a'] as int?; + Uint8List? svgBytes; final int? assetIndex; - final String? svgString; File? svgFile; if (assetIndexJson != null) { if (inlineAssets == null) { svgFile = FileManager.getFile('$sbnPath${Editor.extension}.$assetIndexJson'); } else { - svgString = utf8.decode(inlineAssets[assetIndexJson]); + svgBytes=inlineAssets[assetIndexJson]; + svgFile=assetCacheAll.createRuntimeFile(json['e'] ?? '.svg',svgBytes); } } else if (json['b'] != null) { - svgString = json['b'] as String; + svgBytes = json['b']; + svgFile=assetCacheAll.createRuntimeFile(json['e'] ?? '.svg',svgBytes!); } else { log.warning('SvgEditorImage.fromJson: no svg string found'); - svgString = ''; } if (svgFile != null) { assetIndex = assetCacheAll.addSync(svgFile); @@ -79,8 +74,6 @@ class SvgEditorImage extends EditorImage { -1, // -1 will be replaced by EditorCoreInfo._handleEmptyImageIds() assetCacheAll: assetCacheAll, assetId: assetIndex, - svgString: svgString, - svgFile: svgFile, pageIndex: json['i'] ?? 0, pageSize: Size.infinite, invertible: json['v'] ?? true, @@ -120,7 +113,7 @@ class SvgEditorImage extends EditorImage { assert(!json.containsKey('a')); assert(!json.containsKey('b')); - final svgData = _extractSvg(); +// final svgData = _extractSvg(); json['a'] = assetId; return json; @@ -198,14 +191,12 @@ class SvgEditorImage extends EditorImage { @override SvgEditorImage copy() { - final svgData = _extractSvg(); + //final svgData = _extractSvg(); return SvgEditorImage( id: id, // ignore: deprecated_member_use_from_same_package assetCacheAll: assetCacheAll, assetId: assetId, - svgString: svgData.string, - svgFile: svgData.file, pageIndex: pageIndex, pageSize: Size.infinite, invertible: invertible, From 1b2b781a0be1ffeedc05b6f45e713cb189cdf7a2 Mon Sep 17 00:00:00 2001 From: QubaB Date: Mon, 6 Oct 2025 17:13:05 +0200 Subject: [PATCH 17/35] pdfImage fixed test of assetIndex --- lib/components/canvas/image/pdf_editor_image.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/components/canvas/image/pdf_editor_image.dart b/lib/components/canvas/image/pdf_editor_image.dart index 6640304fa5..43f2fba1ea 100644 --- a/lib/components/canvas/image/pdf_editor_image.dart +++ b/lib/components/canvas/image/pdf_editor_image.dart @@ -74,7 +74,7 @@ class PdfEditorImage extends EditorImage { assetIndex = assetCacheAll.addSync(tempFile); } - assert(assetIndex >0, + assert(assetIndex >=0, 'Either pdfBytes or pdfFile must be non-null'); // add to asset cache From a133e5304187a4a3a88d83865ff00967426cb25d Mon Sep 17 00:00:00 2001 From: QubaB Date: Mon, 6 Oct 2025 17:13:57 +0200 Subject: [PATCH 18/35] Editor - fixed delete, undo/redo to increment and decrement asset use. --- lib/pages/editor/editor.dart | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index b363f62b8a..a4747fa76b 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -366,6 +366,19 @@ class EditorState extends State { for (EditorImage image in item.images) { createPage(image.pageIndex); coreInfo.pages[image.pageIndex].images.add(image); + // increment use of image asset + int assetId=-1; + if (image is PdfEditorImage){ + assetId=(image as PdfEditorImage).assetId; + } else if (image is PngEditorImage){ + assetId=(image as PngEditorImage).assetId; + } else if (image is SvgEditorImage){ + assetId=(image as SvgEditorImage).assetId; + } + if (assetId>=0) { + // free use of asset + image.assetCacheAll.addUse(assetId); + } image.newImage = true; } @@ -767,6 +780,18 @@ class EditorState extends State { } void onDeleteImage(EditorImage image) { + int assetId=-1; + if (image is PdfEditorImage){ + assetId=(image as PdfEditorImage).assetId; + } else if (image is PngEditorImage){ + assetId=(image as PngEditorImage).assetId; + } else if (image is SvgEditorImage){ + assetId=(image as SvgEditorImage).assetId; + } + if (assetId>=0) { + // free use of asset + image.assetCacheAll.removeUse(assetId); + } history.recordChange(EditorHistoryItem( type: EditorHistoryItemType.erase, pageIndex: image.pageIndex, @@ -1083,15 +1108,15 @@ class EditorState extends State { final List images = []; for (final _PhotoInfo photoInfo in photoInfos) { - - + if (photoInfo.extension == '.pdf') { + // pdf can be selected on android, but it cannot be imported + continue; + } if (photoInfo.extension == '.svg') { // add image to assets using its path int assetIndex = await coreInfo.assetCacheAll.add(File(photoInfo.path)); images.add(SvgEditorImage( id: coreInfo.nextImageId++, - svgString: utf8.decode(photoInfo.bytes), - svgFile: null, pageIndex: currentPageIndex, pageSize: coreInfo.pages[currentPageIndex].size, onMoveImage: onMoveImage, From be0b92ac9e832b44976ccfe7b5a325cde1133c94 Mon Sep 17 00:00:00 2001 From: QubaB Date: Mon, 6 Oct 2025 17:14:14 +0200 Subject: [PATCH 19/35] removed unused import --- lib/components/canvas/image/editor_image.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/components/canvas/image/editor_image.dart b/lib/components/canvas/image/editor_image.dart index 63f070862c..743927d0d0 100644 --- a/lib/components/canvas/image/editor_image.dart +++ b/lib/components/canvas/image/editor_image.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; import 'dart:math'; From 764753b8394d6570a8e4b6346f2d0d109017113d Mon Sep 17 00:00:00 2001 From: QubaB Date: Tue, 7 Oct 2025 21:15:35 +0200 Subject: [PATCH 20/35] implemented renumbering of assets. implemented replacement of image provider when asset filename is changed --- lib/components/canvas/_asset_cache.dart | 238 +++++++----------- .../canvas/image/pdf_editor_image.dart | 2 +- .../canvas/image/png_editor_image.dart | 4 +- .../canvas/image/svg_editor_image.dart | 11 +- lib/data/file_manager/file_manager.dart | 4 + lib/pages/editor/editor.dart | 24 +- 6 files changed, 113 insertions(+), 170 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index 323f8e620e..c966cdf968 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -2,12 +2,20 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; import 'dart:math'; -import 'package:collection/collection.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:logging/logging.dart'; import 'package:pdfrx/pdfrx.dart'; -import 'package:saber/components/canvas/image/editor_image.dart'; +import 'package:saber/data/file_manager/file_manager.dart'; + +class RandomFileName { + // generate random file name + static String generateRandomFileName([String extension = 'txt']) { + final timestamp = DateTime.now().millisecondsSinceEpoch; + final random = Random().nextInt(1 << 32); // 32-bit random number + return 'file_${timestamp}_$random.$extension'; + } +} class PdfInfoExtractor { /// Extracts /Info dictionary from a PDF file with minimal reads (synchronous) @@ -252,134 +260,6 @@ class PdfInfoExtractor { } } -/// A cache for assets that are loaded from disk. -/// -/// This is the analogue to Flutter's image cache, -/// but for non-image assets. -/// -/// There should be one instance of this class per -/// [EditorCoreInfo] instance. -class AssetCache { - AssetCache(); - - final log = Logger('AssetCache'); - - /// Maps a file path to its value. - final Map _cache = {}; - - /// Maps a file path to the visible images that use it. - final Map> _images = {}; - - /// Whether items from the cache can be removed: - /// set to false during file save. - bool allowRemovingAssets = true; - - /// Marks [image] as currently visible. - /// - /// It's safe to call this method multiple times. - /// - /// [file] is allowed to be null for convenience, - /// in which case this function does nothing. - void addImage(EditorImage image, File? file, T value) { - if (file == null) return; - _images.putIfAbsent(file.path, () => {}).add(image); - _cache[file.path] = value; - } - - /// Returns null if [file] is not found. - T? get(File file) { - return _cache[file.path] as T?; - } - - /// Marks [image] as no longer visible. - /// - /// It's safe to call this method multiple times. - /// - /// If [image] is the last image using its file, - /// the file is also removed from the cache. - /// - /// Returns whether the image was present in the cache. - bool removeImage(EditorImage image) { - if (!allowRemovingAssets) return false; - - for (final filePath in _images.keys) { - final imagesUsingFile = _images[filePath]!; - imagesUsingFile.remove(image); - - if (imagesUsingFile.isEmpty) { - _images.remove(filePath); - _cache.remove(filePath); - return true; - } - } - - return false; - } - - void dispose() { - _images.clear(); - _cache.clear(); - } -} - -class OrderedAssetCache { - OrderedAssetCache(); - - final log = Logger('OrderedAssetCache'); - - final List _cache = []; - - /// Adds [value] to the cache if it is not already present and - /// returns the index of the added item. - int add(T value) { - int index = _cache.indexOf(value); - if (index == -1 && value is List) { - // Lists need to be compared per item - final listEq = const ListEquality().equals; - for (int i = 0; i < _cache.length; i++) { - final cacheItem = _cache[i]; - if (cacheItem is! List) continue; - if (!listEq(value, cacheItem)) continue; - - index = i; - break; - } - } - log.fine('OrderedAssetCache.add: index = $index, value = $value'); - - if (index == -1) { - _cache.add(value); - return _cache.length - 1; - } else { - return index; - } - } - - /// The number of (distinct) items in the cache. - int get length => _cache.length; - bool get isEmpty => _cache.isEmpty; - bool get isNotEmpty => _cache.isNotEmpty; - - /// Converts the item at position [index] - /// to bytes and returns them. - Future> getBytes(int index) async { - final item = _cache[index]; - if (item is List) { - return item; - } else if (item is File) { - return item.readAsBytes(); - } else if (item is String) { - return utf8.encode(item); - } else if (item is MemoryImage) { - return item.bytes; - } else if (item is FileImage) { - return item.file.readAsBytes(); - } else { - throw Exception( - 'OrderedAssetCache.getBytes: unknown type ${item.runtimeType}'); - } - } -} /////////////////////////////////////////////////////////////////////////// /// current cache and images problems: /// 1. two caches assetCache (for working), OrderedAssetCache (for writing) @@ -435,7 +315,7 @@ class PreviewResult { // object in cache class CacheItem { - final Object value; + Object? value; // value can change because file can be renamed int? previewHash; // quick hash (from first 100KB bytes) int? hash; // hash can be calculated later String? fileInfo; // file information - /Info of pdf is implemented now @@ -446,11 +326,10 @@ class CacheItem { // for files only final int? fileSize; - final String? - filePath; // only for files - for fast comparison without reading file contents + String? filePath; // only for files - for fast comparison without reading file contents final String? fileExt; // file extension int _refCount = 0; // number of references - bool _released = false; + int assetIdOnSave=-1; // id of asset when the note is save (some assets can be skipped) CacheItem( this.value, { @@ -466,18 +345,15 @@ class CacheItem { // increase use of item void addUse() { - //if (_released) throw StateError('Trying to add use of released CacheItem'); _refCount++; } // when asset is released (no more used) void freeUse() { if (_refCount > 0) _refCount--; - //if (_refCount == 0) _released = true; } - + int get refCount => _refCount; bool get isUnused => _refCount == 0; - bool get isReleased => _released; @override bool operator ==(Object other) { @@ -494,7 +370,7 @@ class CacheItem { } } - if (fileInfo != null && other.fileInfo != null) { + if (fileInfo != '' && other.fileInfo != '') { if (fileInfo == other.fileInfo) { // both file info are the same. return true; @@ -549,12 +425,22 @@ class CacheItem { imageProviderNotifier.value = null; // will be recreated on next access } + // move asset file to temporary file to avoid it is overwriten during note save + void moveAssetToTemporaryFile() { + final dir = FileManager.supportDirectory; + String newPath = '${dir.path}/TmPmP_${RandomFileName.generateRandomFileName(fileExt != null ? fileExt! : 'tmp')}'; // update file path + value=(value as File).renameSync(newPath); // rename asset + filePath=newPath; + } + + + // @override // int? get hash => filePath?.hash ?? hash; @override String toString() => - 'CacheItem(path: $filePath, preview=$previewHash, full=$hash, refs=$_refCount, released=$_released)'; + 'CacheItem(path: $filePath, preview=$previewHash, full=$hash, refs=$_refCount)'; } // cache manager @@ -574,6 +460,7 @@ class AssetCacheAll { final log = Logger('OrderedAssetCache'); + // return pdfDocument of asset it is lazy because it take some time to do it Future getPdfDocument(int assetId) { // if already opened, return it immediately @@ -608,6 +495,24 @@ class AssetCacheAll { } } + + // removes image provider of file from image Cache. + // important to call this when assets are renamed + Future clearImageProvider(int assetId) async { + // return cached provider if already available + final item = _items[assetId]; + if (item.imageProviderNotifier.value == null) + return; + if (item.value is File) { + final file=FileImage(item.value as File); + final key = await file.obtainKey(ImageConfiguration()); + imageCache.evict(key); + } + return; + } + + + // give image provider notifier for asset image ValueNotifier getImageProviderNotifier(int assetId) { // return cached provider if already available @@ -844,6 +749,50 @@ class AssetCacheAll { } } + // return id of asset which must be used on save + int getAssetIdOnSave(int index){ + return _items[index].assetIdOnSave; + } + + // this routine should be called before note is saved. It renumbers assets according their usage + Future renumberBeforeSave(String noteFilePath) async{ + int currentId=-1; + for (int i = 0; i < _items.length; i++) { + final item = _items[i]; + if (item.refCount > 0) { + currentId++; + item.assetIdOnSave = currentId; + if (item.assetIdOnSave != i) { + final newPath = '$noteFilePath.$currentId'; // new asset file + if ((item.filePath) != newPath) { + // move asset to correct file name + await clearImageProvider(i); // remove imageProvider from cache + item.value = + (item.value as File).renameSync(newPath); // rename asset + item.filePath = newPath; + item.invalidateImageProvider(); // invalidate image provider so new one is allocated + getImageProviderNotifier(i); // allocate new image provider to enable rendering + } + } + } + else { + // this item is not used. We should save it to temp directory + // in case of undo it can be used again + item.assetIdOnSave = -1; + if ((item.filePath)!.startsWith(noteFilePath)){ + // file path of asset is the same as note path - asset file can be overwritten, + // move it to the safe location + await clearImageProvider(i); // remove imageProvider from cache + item.moveAssetToTemporaryFile(); // move asset to different file + item.invalidateImageProvider(); // invalidate image provider so new one is allocated + getImageProviderNotifier(i); // allocate new image provider to enable rendering + } + } + } + return; + } + + // finalize cache after it was filled using addSync - without calculation of hashes // is called after note is read to Editor Future finalize() async { @@ -919,18 +868,11 @@ class AssetCacheAll { } } - // generate random file name - String generateRandomFileName([String extension = 'txt']) { - final timestamp = DateTime.now().millisecondsSinceEpoch; - final random = Random().nextInt(1 << 32); // 32-bit random number - return 'file_${timestamp}_$random.$extension'; - } - // create temporary file from bytes when inline bytes are read File createRuntimeFile(String ext, Uint8List bytes) { final dir = Directory.systemTemp; // Použití systémového temp adresáře // final dir = await getApplicationSupportDirectory(); - final file = File('${dir.path}/TmPmP_${generateRandomFileName(ext)}'); + final file = File('${dir.path}/TmPmP_${RandomFileName.generateRandomFileName(ext)}'); file.writeAsBytesSync(bytes, flush: true); return file; } diff --git a/lib/components/canvas/image/pdf_editor_image.dart b/lib/components/canvas/image/pdf_editor_image.dart index 43f2fba1ea..a59eae9713 100644 --- a/lib/components/canvas/image/pdf_editor_image.dart +++ b/lib/components/canvas/image/pdf_editor_image.dart @@ -122,7 +122,7 @@ class PdfEditorImage extends EditorImage { assert(!json.containsKey('a')); assert(!json.containsKey('b')); - json['a'] = assetId; + json['a'] = assetCacheAll.getAssetIdOnSave(assetId); // assets can be reordered during saving json['pdfi'] = pdfPage; return json; diff --git a/lib/components/canvas/image/png_editor_image.dart b/lib/components/canvas/image/png_editor_image.dart index c96c306201..4dfdcddd0c 100644 --- a/lib/components/canvas/image/png_editor_image.dart +++ b/lib/components/canvas/image/png_editor_image.dart @@ -140,7 +140,7 @@ class PngEditorImage extends EditorImage { @override Map toJson() => super.toJson() ..addAll({ - if (imageProvider != null) 'a': assetId, + 'a': assetCacheAll.getAssetIdOnSave(assetId), // assets can be reordered during saving }); @override @@ -179,7 +179,7 @@ class PngEditorImage extends EditorImage { ); if (resizedByteData != null) { // store resized bytes to temporary file - final tempImageFile = await assetCacheAll.createRuntimeFile('.png',resizedByteData.buffer.asUint8List()); + final tempImageFile = assetCacheAll.createRuntimeFile('.png',resizedByteData.buffer.asUint8List()); // replace image assetCacheAll.replaceImage(tempImageFile, assetId); } diff --git a/lib/components/canvas/image/svg_editor_image.dart b/lib/components/canvas/image/svg_editor_image.dart index 62d17981de..a2a41df94a 100644 --- a/lib/components/canvas/image/svg_editor_image.dart +++ b/lib/components/canvas/image/svg_editor_image.dart @@ -114,20 +114,11 @@ class SvgEditorImage extends EditorImage { assert(!json.containsKey('b')); // final svgData = _extractSvg(); - json['a'] = assetId; + json['a'] = assetCacheAll.getAssetIdOnSave(assetId); // assets can be reordered during saving return json; } - ({String? string, File? file}) _extractSvg() => switch (svgLoader) { - (SvgStringLoader loader) => ( - string: loader.provideSvg(null), - file: null - ), - (SvgFileLoader loader) => (string: null, file: loader.file), - (_) => throw ArgumentError.value(svgLoader, 'svgLoader', - 'SvgEditorImage.toJson: svgLoader must be a SvgStringLoader or SvgFileLoader'), - }; @override Future firstLoad() async { diff --git a/lib/data/file_manager/file_manager.dart b/lib/data/file_manager/file_manager.dart index ffb1c56d7f..e70db8e4f2 100644 --- a/lib/data/file_manager/file_manager.dart +++ b/lib/data/file_manager/file_manager.dart @@ -31,6 +31,9 @@ class FileManager { /// Realistically, this value never changes. static late String documentsDirectory; + // Support directory is for storing intermediate files (these files can be cleaned after note is saved) + static late Directory supportDirectory; + static final fileWriteStream = StreamController.broadcast(); // TODO(adil192): Implement or remove this @@ -46,6 +49,7 @@ class FileManager { }) async { FileManager.documentsDirectory = documentsDirectory ?? await getDocumentsDirectory(); + FileManager.supportDirectory = await getApplicationSupportDirectory(); if (shouldWatchRootDirectory) unawaited(watchRootDirectory()); } diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index a4747fa76b..93168321f9 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'dart:convert'; import 'dart:io'; import 'package:collapsible/collapsible.dart'; @@ -13,7 +12,6 @@ import 'package:flutter_quill/flutter_quill.dart' as flutter_quill; import 'package:keybinder/keybinder.dart'; import 'package:logging/logging.dart'; import 'package:printing/printing.dart'; -import 'package:saber/components/canvas/_asset_cache.dart'; import 'package:saber/components/canvas/_stroke.dart'; import 'package:saber/components/canvas/canvas.dart'; import 'package:saber/components/canvas/canvas_gesture_detector.dart'; @@ -369,11 +367,11 @@ class EditorState extends State { // increment use of image asset int assetId=-1; if (image is PdfEditorImage){ - assetId=(image as PdfEditorImage).assetId; + assetId=image.assetId; } else if (image is PngEditorImage){ - assetId=(image as PngEditorImage).assetId; + assetId=image.assetId; } else if (image is SvgEditorImage){ - assetId=(image as SvgEditorImage).assetId; + assetId=image.assetId; } if (assetId>=0) { // free use of asset @@ -782,11 +780,11 @@ class EditorState extends State { void onDeleteImage(EditorImage image) { int assetId=-1; if (image is PdfEditorImage){ - assetId=(image as PdfEditorImage).assetId; + assetId=image.assetId; } else if (image is PngEditorImage){ - assetId=(image as PngEditorImage).assetId; + assetId=image.assetId; } else if (image is SvgEditorImage){ - assetId=(image as SvgEditorImage).assetId; + assetId=image.assetId; } if (assetId>=0) { // free use of asset @@ -929,6 +927,8 @@ class EditorState extends State { final filePath = coreInfo.filePath + Editor.extension; final Uint8List bson; coreInfo.assetCacheAll.allowRemovingAssets = false; + final fullPath = FileManager.getFilePath(filePath); // add full path of note + await coreInfo.assetCacheAll.renumberBeforeSave(fullPath); // renumber all assets try { // go through all pages and prepare Json of each page. (bson) = coreInfo.saveToBinary( @@ -943,8 +943,14 @@ class EditorState extends State { // write assets for (int i = 0; i < coreInfo.assetCacheAll.length; ++i){ + final idSave=coreInfo.assetCacheAll.getAssetIdOnSave(i); + if (idSave<0){ + // asset is not used + continue; + } final assetFile=coreInfo.assetCacheAll.getAssetFile(i); - await FileManager.copyFile(assetFile,'$filePath.$i', awaitWrite: true); + final newFile='$filePath.$idSave'; // new asset file + await FileManager.copyFile(assetFile,newFile, awaitWrite: true); } FileManager.removeUnusedAssets( filePath, From 5774b1f962ee9105f6fa7e7991d99f206be7bd8c Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 8 Oct 2025 08:32:27 +0200 Subject: [PATCH 21/35] assets are copied (renamed) to correct file names in renumberBeforeSave so after writing note only markFileAsSaved is called to upload items, ... --- lib/data/file_manager/file_manager.dart | 20 ++++++++++++++++++++ lib/pages/editor/editor.dart | 11 ++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/lib/data/file_manager/file_manager.dart b/lib/data/file_manager/file_manager.dart index e70db8e4f2..a34c8e9ee8 100644 --- a/lib/data/file_manager/file_manager.dart +++ b/lib/data/file_manager/file_manager.dart @@ -309,6 +309,26 @@ class FileManager { if (awaitWrite) await writeFuture; } + /// Marks [fileFrom] as saved - used when saving assets during note save. + static Future markFileAsSaved( + File fileFrom, + { + bool awaitWrite = false, + bool alsoUpload = true, + }) async { + log.fine('Marking file as Saved'); + + await _saveFileAsRecentlyAccessed(fileFrom.path); + broadcastFileWrite(FileOperationType.write, fileFrom.path); + if (alsoUpload) syncer.uploader.enqueueRel(fileFrom.path); + if (fileFrom.path.endsWith(Editor.extension)) { + _removeReferences( + '${fileFrom.path.substring(0, fileFrom.path.length - Editor.extension.length)}' + '${Editor.extensionOldJson}'); + } + } + + static Future createFolder(String folderPath) async { folderPath = _sanitisePath(folderPath); diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index 93168321f9..3739008308 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -941,20 +941,21 @@ class EditorState extends State { // write note itself await FileManager.writeFile(filePath, bson, awaitWrite: true); - // write assets + // write assets was already done in renumberBeforeSave() routine + int numAssetUsed=0; for (int i = 0; i < coreInfo.assetCacheAll.length; ++i){ final idSave=coreInfo.assetCacheAll.getAssetIdOnSave(i); if (idSave<0){ - // asset is not used + // asset is not used do not save it continue; } + numAssetUsed++; // increase number of used assets final assetFile=coreInfo.assetCacheAll.getAssetFile(i); - final newFile='$filePath.$idSave'; // new asset file - await FileManager.copyFile(assetFile,newFile, awaitWrite: true); + await FileManager.markFileAsSaved(assetFile); // inform application that file was updated } FileManager.removeUnusedAssets( filePath, - numAssets: coreInfo.assetCacheAll.length, + numAssets: numAssetUsed, ); savingState.value = SavingState.saved; } catch (e) { From 4cd379383a6e4ac19f929d4bd0f839a16878cd42 Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 8 Oct 2025 08:50:20 +0200 Subject: [PATCH 22/35] importing pdf increase asset use counter for each page --- lib/pages/editor/editor.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index 3739008308..ce0b5f89ac 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -1270,6 +1270,7 @@ class EditorState extends State { assetCacheAll: coreInfo.assetCacheAll, assetId: assetIndex, ); + coreInfo.assetCacheAll.addUse(assetIndex); // inform that asset is uded more times coreInfo.pages.add(page); history.recordChange(EditorHistoryItem( type: EditorHistoryItemType.insertPage, From a8c1e14ea4ccb638cd1f5db85e0de1712a69f4dc Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 8 Oct 2025 08:51:07 +0200 Subject: [PATCH 23/35] in json is stored also fileInfo, previewHash, Hash, fileSize to increase speed of loading assets --- lib/components/canvas/_asset_cache.dart | 313 +++++++++++------- .../canvas/image/pdf_editor_image.dart | 27 +- .../canvas/image/png_editor_image.dart | 27 +- .../canvas/image/svg_editor_image.dart | 14 +- 4 files changed, 245 insertions(+), 136 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index c966cdf968..b24eac32c1 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -5,15 +5,16 @@ import 'dart:math'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:logging/logging.dart'; +import 'package:mutex/mutex.dart'; import 'package:pdfrx/pdfrx.dart'; import 'package:saber/data/file_manager/file_manager.dart'; class RandomFileName { // generate random file name - static String generateRandomFileName([String extension = 'txt']) { + static String generateRandomFileName([String extension = '.txt']) { final timestamp = DateTime.now().millisecondsSinceEpoch; final random = Random().nextInt(1 << 32); // 32-bit random number - return 'file_${timestamp}_$random.$extension'; + return 'file_${timestamp}_$random$extension'; } } @@ -308,9 +309,8 @@ class PdfInfoExtractor { class PreviewResult { final int previewHash; final int fileSize; - final String firstBytes; final String fileInfo; // string containing file information - now only pdfFile /Info - PreviewResult(this.previewHash, this.fileSize, this.firstBytes, this.fileInfo); + PreviewResult(this.previewHash, this.fileSize, this.fileInfo); } // object in cache @@ -454,6 +454,8 @@ class AssetCacheAll { final Map> _openingDocs = {}; // Holds currently opening futures to avoid duplicate opens + final Mutex _mutex = Mutex(); // blocking some operations + /// Whether items from the cache can be removed: /// set to false during file save. bool allowRemovingAssets = true; @@ -511,8 +513,6 @@ class AssetCacheAll { return; } - - // give image provider notifier for asset image ValueNotifier getImageProviderNotifier(int assetId) { // return cached provider if already available @@ -536,9 +536,23 @@ class AssetCacheAll { } // calculate hash of bytes (all) - int calculateHash(List bytes) { + int calculateHash(List bytes,int fileSize) { // fnv1a int hash = 0x811C9DC5; + + // first 4 bytes will be hashed file size + hash ^= (fileSize & 0xFF); // lowest byte + hash = (hash * 0x01000193) & 0xFFFFFFFF; + + hash ^= ((fileSize >> 8) & 0xFF); // second byte + hash = (hash * 0x01000193) & 0xFFFFFFFF; + + hash ^= ((fileSize >> 16) & 0xFF); // third byte + hash = (hash * 0x01000193) & 0xFFFFFFFF; + + hash ^= ((fileSize >> 24) & 0xFF); // fourth byte + + // and hash of filebytes for (var b in bytes) { hash ^= b; hash = (hash * 0x01000193) & 0xFFFFFFFF; @@ -549,7 +563,7 @@ class AssetCacheAll { // Compute a quick hash based on the first 100 KB of the file. // This can be done synchronously to quickly filter duplicates. // calculate preview hash of file - PreviewResult getFilePreviewHash(File file) { + PreviewResult getFilePreviewHash(File file,String extension) { final stat = file.statSync(); // get file metadata final fileSize = stat.size; @@ -558,10 +572,9 @@ class AssetCacheAll { // read either the whole file if small, or just the first 100 KB final toRead = fileSize < 100 * 1024 ? fileSize : 100 * 1024; final bytes = raf.readSync(toRead); - final previewHashBytes = calculateHash(bytes); - final firstBytes =latin1.decode(bytes.sublist(0, 4)); // first 4 characters - used to detect PDF file + final previewHashBytes = calculateHash(bytes,fileSize); String fileInfo=''; - if (firstBytes == '%PDF') { + if (extension == '.pdf') { // asset is pdf, get pdf /Info, it is quick try { final info = PdfInfoExtractor.extractInfo(file); @@ -571,11 +584,9 @@ class AssetCacheAll { fileInfo=''; } } - return PreviewResult( - (fileSize.hashCode << 32) ^ previewHashBytes, // previehash is put together from file size and hash of first 100kB + previewHashBytes, // previehash is put together from file size and hash of first 100kB fileSize, // file size - firstBytes, // first 4 bytes - used to recognize pdf file format fileInfo ); } finally { @@ -601,7 +612,12 @@ class AssetCacheAll { // It is why we simply not add every asset, because they can be the same. // add to cache but read only small part of files - used when reading note from disk // full hashes are established later - int addSync(Object value) { + int addSync(Object value, + String extension, + String? fileInfo, + int? previewHash, + int? fileSize, + int? hash) { if (value is File) { log.info('allCache.addSync: value = $value'); final path = value.path; @@ -614,9 +630,14 @@ class AssetCacheAll { 'allCache.addSync: already in cache {$_items[existingPathIndex]._refCount}'); return existingPathIndex; } - - final previewResult = - getFilePreviewHash(value); // calculate preliminary hash of file or get Info of pdf file + PreviewResult previewResult; + if (previewHash != null && fileSize !=null){ + // file information is stored directly in asset no need to determine it + previewResult = PreviewResult(previewHash, fileSize, fileInfo ?? ''); + } + else { + previewResult =getFilePreviewHash(value,extension); // calculate preliminary hash of file or get Info of pdf file + } if (previewResult.fileInfo.isNotEmpty){ // we know information about file (it is pdf) so test it @@ -642,7 +663,9 @@ class AssetCacheAll { filePath: value.path, previewHash: previewResult.previewHash, fileSize: previewResult.fileSize, - fileInfo: previewResult.fileInfo) + fileExt: extension, + fileInfo: previewResult.fileInfo, + hash: hash) ..addUse(); // and add use _items.add(newItem); @@ -659,67 +682,72 @@ class AssetCacheAll { // async add cache is used from Editor, when adding asset using file picker // always is used File! Future add(Object value) async { - if (value is File) { - // files are first compared by file path - final path = value.path; - - // 1. Fast path check - final existingPathIndex = _items.indexWhere((i) => i.filePath == path); - if (existingPathIndex != -1) { - _items[existingPathIndex].addUse(); - return existingPathIndex; - } - - final previewResult = - getFilePreviewHash(value); // calculate preliminary hash of file - - if (previewResult.fileInfo.isNotEmpty){ - // we know information about file (pdf file) so test it - final existingFileInfoIndex = _items.indexWhere((i) => i.fileInfo == previewResult.fileInfo); - if (existingFileInfoIndex != -1) { - // file with this fileInfo already in cache - _items[existingFileInfoIndex].addUse(); - log.info( - 'allCache.addSync: already in cache {$_items[existingPathIndex]._refCount}'); - return existingFileInfoIndex; + return await _mutex.protect(() async { + if (value is File) { + // files are first compared by file path + final path = value.path; + final extension = '.${value.path.split('.').last.toLowerCase()}'; + + // 1. Fast path check + final existingPathIndex = _items.indexWhere((i) => i.filePath == path); + if (existingPathIndex != -1) { + _items[existingPathIndex].addUse(); + return existingPathIndex; } - } - // Check previwHash value - if (_previewHashIndex.containsKey(previewResult.previewHash)) { - final existingIndex = _previewHashIndex[previewResult.previewHash]!; - _items[existingIndex].addUse(); - return existingIndex; - } + final previewResult = + getFilePreviewHash(value,extension); // calculate preliminary hash of file + + if (previewResult.fileInfo.isNotEmpty) { + // we know information about file (pdf file) so test it + final existingFileInfoIndex = _items.indexWhere((i) => + i.fileInfo == previewResult.fileInfo); + if (existingFileInfoIndex != -1) { + // file with this fileInfo already in cache + _items[existingFileInfoIndex].addUse(); + log.info( + 'allCache.addSync: already in cache {$_items[existingPathIndex]._refCount}'); + return existingFileInfoIndex; + } + } - // compute expensive content hash - need to read whole file - final bytes = await value.readAsBytes(); - final hash = calculateHash(bytes); + // Check previwHash value + if (_previewHashIndex.containsKey(previewResult.previewHash)) { + final existingIndex = _previewHashIndex[previewResult.previewHash]!; + _items[existingIndex].addUse(); + return existingIndex; + } - // prepare cache item - final newItem = CacheItem(value, - filePath: value.path, - previewHash: previewResult.previewHash, - hash: hash, - fileSize: previewResult.fileSize, - fileInfo: previewResult.fileInfo) - ..addUse(); - - // check if it is already in cache - final existingHashIndex = _items.indexOf(newItem); - if (existingHashIndex != -1) { - _items[existingHashIndex].addUse(); - return existingHashIndex; + // compute expensive content hash - need to read whole file + final bytes = await value.readAsBytes(); + final hash = calculateHash(bytes,previewResult.fileSize); + + // prepare cache item + final newItem = CacheItem(value, + filePath: value.path, + previewHash: previewResult.previewHash, + hash: hash, + fileExt: extension, + fileSize: previewResult.fileSize, + fileInfo: previewResult.fileInfo) + ..addUse(); + + // check if it is already in cache + final existingHashIndex = _items.indexOf(newItem); + if (existingHashIndex != -1) { + _items[existingHashIndex].addUse(); + return existingHashIndex; + } + _items.add(newItem); + final index = _items.length - 1; + _previewHashIndex[previewResult.previewHash] = + index; // add to previously hashed + return index; + } else { + throw Exception( + 'assetCacheAll.add: unknown type ${value.runtimeType}'); } - _items.add(newItem); - final index = _items.length - 1; - _previewHashIndex[previewResult.previewHash] = - index; // add to previously hashed - return index; - } else{ - throw Exception( - 'assetCacheAll.add: unknown type ${value.runtimeType}'); - } + }); } /// The number of (distinct) items in the cache. @@ -754,42 +782,69 @@ class AssetCacheAll { return _items[index].assetIdOnSave; } + // return asset previewHash + int getAssetPreviewHash(int index){ + return _items[index].previewHash!; + } + + // return asset Hash + int? getAssetHash(int index){ + return _items[index].hash; + } + + // return asset fileInfo + String getAssetFileInfo(int index){ + return _items[index].fileInfo ?? ''; + } + + // return asset fileSize + int getAssetFileSize(int index){ + return _items[index].fileSize!; + } + // this routine should be called before note is saved. It renumbers assets according their usage Future renumberBeforeSave(String noteFilePath) async{ - int currentId=-1; - for (int i = 0; i < _items.length; i++) { - final item = _items[i]; - if (item.refCount > 0) { - currentId++; - item.assetIdOnSave = currentId; - if (item.assetIdOnSave != i) { + await _mutex.protect(() async { + int currentId = -1; + for (int i = 0; i < _items.length; i++) { + final item = _items[i]; + if (item.refCount > 0) { + currentId++; final newPath = '$noteFilePath.$currentId'; // new asset file + item.assetIdOnSave = currentId; if ((item.filePath) != newPath) { // move asset to correct file name - await clearImageProvider(i); // remove imageProvider from cache + final bool isImage = item.imageProviderNotifier !=null; + final bool isPdf = item._pdfDocument !=null; + if (isImage) { + await clearImageProvider(i); // remove imageProvider from cache + } else if (isPdf){ + + } + item.value = (item.value as File).renameSync(newPath); // rename asset item.filePath = newPath; item.invalidateImageProvider(); // invalidate image provider so new one is allocated - getImageProviderNotifier(i); // allocate new image provider to enable rendering + getImageProviderNotifier(i); // allocate new image provider to enable rendering } } - } - else { - // this item is not used. We should save it to temp directory - // in case of undo it can be used again - item.assetIdOnSave = -1; - if ((item.filePath)!.startsWith(noteFilePath)){ - // file path of asset is the same as note path - asset file can be overwritten, - // move it to the safe location - await clearImageProvider(i); // remove imageProvider from cache - item.moveAssetToTemporaryFile(); // move asset to different file - item.invalidateImageProvider(); // invalidate image provider so new one is allocated - getImageProviderNotifier(i); // allocate new image provider to enable rendering + else { + // this item is not used. We should save it to temp directory + // in case of undo it can be used again + item.assetIdOnSave = -1; + if ((item.filePath)!.startsWith(noteFilePath)) { + // file path of asset is the same as note path - asset file can be overwritten, + // move it to the safe location + await clearImageProvider(i); // remove imageProvider from cache + item.moveAssetToTemporaryFile(); // move asset to different file + item.invalidateImageProvider(); // invalidate image provider so new one is allocated + getImageProviderNotifier(i); // allocate new image provider to enable rendering + } } } - } - return; + return; + }); } @@ -803,7 +858,7 @@ class AssetCacheAll { int? hashItem = item.hash; if (hashItem == 0) { final bytes = await getBytes(i); - hash = calculateHash(bytes); + hash = calculateHash(bytes,_items[i].fileSize!); _items[i] = CacheItem(item.value, hash: hash, filePath: item.filePath, fileSize: item.fileSize); } else { @@ -826,33 +881,36 @@ class AssetCacheAll { // replace asset by another one - typically when resampling image to lower resolution Future replaceImage(Object value, int id) async { - if (value is File) { - // compute expensive content hash - final bytes = await value.readAsBytes(); - final hash = calculateHash(bytes); - final previewResult = - getFilePreviewHash(value); // calculate preliminary hash of file - - final oldItem = _items[id]; - // create new Cache item - final newItem = CacheItem( - value, - filePath: value.path, - previewHash: previewResult.previewHash, - hash: hash, - fileSize: previewResult.fileSize, - imageProviderNotifier: - oldItem.imageProviderNotifier, // keep original Notifier - ).._refCount = oldItem._refCount; // keep number of references - - // update original fields - _items[id] = newItem; - _items[id] - .invalidateImageProvider; // invalidate imageProvider so it is newly created when needed - } else { - throw Exception( - 'assetCacheAll.replaceImage: unknown type ${value.runtimeType}'); - } + await _mutex.protect(() async { + if (value is File) { + // compute expensive content hash + final bytes = await value.readAsBytes(); + final previewResult = + getFilePreviewHash(value,_items[id].fileExt!); // calculate preliminary hash of file + final hash = calculateHash(bytes,previewResult.fileSize); + + final oldItem = _items[id]; + // create new Cache item + final newItem = CacheItem( + value, + filePath: value.path, + previewHash: previewResult.previewHash, + hash: hash, + fileSize: previewResult.fileSize, + imageProviderNotifier: + oldItem.imageProviderNotifier, // keep original Notifier + ) + .._refCount = oldItem._refCount; // keep number of references + + // update original fields + _items[id] = newItem; + _items[id] + .invalidateImageProvider; // invalidate imageProvider so it is newly created when needed + } else { + throw Exception( + 'assetCacheAll.replaceImage: unknown type ${value.runtimeType}'); + } + }); } // return File associated with asset, used to save assets when saving note @@ -879,6 +937,9 @@ class AssetCacheAll { void dispose() { _items.clear(); + _aliasMap.clear(); + _openingDocs.clear(); + _previewHashIndex.clear(); } @override diff --git a/lib/components/canvas/image/pdf_editor_image.dart b/lib/components/canvas/image/pdf_editor_image.dart index a59eae9713..1b76d34f58 100644 --- a/lib/components/canvas/image/pdf_editor_image.dart +++ b/lib/components/canvas/image/pdf_editor_image.dart @@ -52,6 +52,7 @@ class PdfEditorImage extends EditorImage { assert(extension == null || extension == '.pdf'); final assetIndexJson = json['a'] as int?; + final Uint8List? pdfBytes; int? assetIndex; File? pdfFile; @@ -59,19 +60,29 @@ class PdfEditorImage extends EditorImage { if (inlineAssets == null) { pdfFile = FileManager.getFile('$sbnPath${Editor.extension}.$assetIndexJson'); - assetIndex = assetCacheAll.addSync(pdfFile); + assetIndex = assetCacheAll.addSync( + pdfFile,'pdf', + json.containsKey('ainf') ? json['ainf'] : null, + json.containsKey('aph') ? json['aph'].toInt() : null, + json.containsKey('afs') ? json['afs'] : null, + json.containsKey('ah') ? json['ah'].toInt() : null, + ); } else { pdfBytes = inlineAssets[assetIndexJson]; final tempFile=assetCacheAll.createRuntimeFile('.pdf',pdfBytes); // store to file - assetIndex = assetCacheAll.addSync(tempFile); + assetIndex = assetCacheAll.addSync( + tempFile,'pdf', + json.containsKey('ainf') ? json['ainf'] : null, + json.containsKey('aph') ? json['aph'].toInt() : null, + json.containsKey('afs') ? json['afs'] : null, + json.containsKey('ah') ? json['ah'].toInt() : null, + ); } } else { if (kDebugMode) { throw Exception('PdfEditorImage.fromJson: pdf bytes not found'); } - pdfBytes = Uint8List(0); - final tempFile=assetCacheAll.createRuntimeFile('.pdf',pdfBytes); - assetIndex = assetCacheAll.addSync(tempFile); + assetIndex=-1; } assert(assetIndex >=0, @@ -124,6 +135,12 @@ class PdfEditorImage extends EditorImage { json['a'] = assetCacheAll.getAssetIdOnSave(assetId); // assets can be reordered during saving json['pdfi'] = pdfPage; + json['aph'] = assetCacheAll.getAssetPreviewHash(assetId); // assets prewiewHash + json['afs'] = assetCacheAll.getAssetFileSize(assetId); // assets can be reordered during saving + if (assetCacheAll.getAssetFileInfo(assetId) != '') + json['ainf'] = assetCacheAll.getAssetFileInfo(assetId); // asset file info + if (assetCacheAll.getAssetHash(assetId) != null) + json['ah'] = assetCacheAll.getAssetHash(assetId); // assets can be reordered during saving return json; } diff --git a/lib/components/canvas/image/png_editor_image.dart b/lib/components/canvas/image/png_editor_image.dart index 4dfdcddd0c..d9c082e680 100644 --- a/lib/components/canvas/image/png_editor_image.dart +++ b/lib/components/canvas/image/png_editor_image.dart @@ -62,6 +62,7 @@ class PngEditorImage extends EditorImage { required AssetCacheAll assetCacheAll, }) { final assetIndexJson = json['a'] as int?; + String? ext=json['e'] ?? '.jpg'; Uint8List? bytes; final int? assetIndex; File? imageFile; @@ -85,11 +86,23 @@ class PngEditorImage extends EditorImage { // add to asset cache if (imageFile != null) { - assetIndex = assetCacheAll.addSync(imageFile); + assetIndex = assetCacheAll.addSync( + imageFile,ext!, + json.containsKey('ainf') ? json['ainf'] : null, + json.containsKey('aph') ? json['aph'].toInt() : null, + json.containsKey('afs') ? json['afs'] : null, + json.containsKey('ah') ? json['ah'].toInt() : null, + ); } else { - final tempFile=assetCacheAll.createRuntimeFile(json['e'] ?? '.jpg',bytes!); - assetIndex = assetCacheAll.addSync(tempFile); + final tempFile=assetCacheAll.createRuntimeFile(ext!,bytes!); + assetIndex = assetCacheAll.addSync( + tempFile,ext, + json.containsKey('ainf') ? json['ainf'] : null, + json.containsKey('aph') ? json['aph'].toInt() : null, + json.containsKey('afs') ? json['afs'] : null, + json.containsKey('ah') ? json['ah'].toInt() : null, + ); } if (assetIndex<0){ throw Exception('EditorImage.fromJson: image not in assets'); @@ -102,7 +115,7 @@ class PngEditorImage extends EditorImage { id: json['id'] ?? -1, assetCacheAll: assetCacheAll, assetId: assetIndex, - extension: json['e'] ?? '.jpg', + extension: ext, imageProviderNotifier: assetCacheAll.getImageProviderNotifier(assetIndex), pageIndex: json['i'] ?? 0, pageSize: Size.infinite, @@ -141,6 +154,12 @@ class PngEditorImage extends EditorImage { Map toJson() => super.toJson() ..addAll({ 'a': assetCacheAll.getAssetIdOnSave(assetId), // assets can be reordered during saving + 'aph': assetCacheAll.getAssetPreviewHash(assetId), // assets prewiewHash + 'afs': assetCacheAll.getAssetFileSize(assetId), // assets can be reordered during saving + if (assetCacheAll.getAssetFileInfo(assetId) != '') + 'ainf': assetCacheAll.getAssetFileInfo(assetId), // asset file info + if (assetCacheAll.getAssetHash(assetId) != null) + 'ah': assetCacheAll.getAssetHash(assetId), // assets can be reordered during saving }); @override diff --git a/lib/components/canvas/image/svg_editor_image.dart b/lib/components/canvas/image/svg_editor_image.dart index a2a41df94a..c924d028d3 100644 --- a/lib/components/canvas/image/svg_editor_image.dart +++ b/lib/components/canvas/image/svg_editor_image.dart @@ -60,7 +60,13 @@ class SvgEditorImage extends EditorImage { log.warning('SvgEditorImage.fromJson: no svg string found'); } if (svgFile != null) { - assetIndex = assetCacheAll.addSync(svgFile); + assetIndex = assetCacheAll.addSync( + svgFile,'svg', + json.containsKey('ainf') ? json['ainf'] : null, + json.containsKey('aph') ? json['aph'].toInt() : null, + json.containsKey('afs') ? json['afs'] : null, + json.containsKey('ah') ? json['ah'].toInt() : null, + ); } else { throw Exception('EditorImage.fromJson: svg image not in assets'); @@ -115,6 +121,12 @@ class SvgEditorImage extends EditorImage { // final svgData = _extractSvg(); json['a'] = assetCacheAll.getAssetIdOnSave(assetId); // assets can be reordered during saving + json['aph'] = assetCacheAll.getAssetPreviewHash(assetId); // assets prewiewHash + json['afs'] = assetCacheAll.getAssetFileSize(assetId); // assets can be reordered during saving + if (assetCacheAll.getAssetFileInfo(assetId) != '') + json['ainf'] = assetCacheAll.getAssetFileInfo(assetId); // asset file info + if (assetCacheAll.getAssetHash(assetId) != null) + json['ah'] = assetCacheAll.getAssetHash(assetId); // assets can be reordered during saving return json; } From 2095e5b7c0e2da6786dfea7baebcbc581c28ad61 Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 8 Oct 2025 10:53:14 +0200 Subject: [PATCH 24/35] getPdfNotifier implemented to notify pdfs about its pdfDocument renderer. It will later allow to change notifier when asset filename will change --- lib/components/canvas/_asset_cache.dart | 49 +++++++++++-------- .../canvas/image/pdf_editor_image.dart | 20 ++------ 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index b24eac32c1..87f7e230f8 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -319,9 +319,13 @@ class CacheItem { int? previewHash; // quick hash (from first 100KB bytes) int? hash; // hash can be calculated later String? fileInfo; // file information - /Info of pdf is implemented now - final ValueNotifier - imageProviderNotifier; // image provider for png, svg as value listener + + final ValueNotifier imageProviderNotifier; // image provider for png, svg as value listener + + PdfDocument? _pdfDocument; // pdf document provider for pdf + final ValueNotifier pdfDocumentNotifier; // pdfDocument as value listener + Uint8List? bytes; // used only when reading inline assets // for files only @@ -340,8 +344,10 @@ class CacheItem { this.fileExt, this.fileInfo, this.bytes, + ValueNotifier? pdfDocumentNotifier, ValueNotifier? imageProviderNotifier, - }) : imageProviderNotifier = imageProviderNotifier ?? ValueNotifier(null); + }) : pdfDocumentNotifier = pdfDocumentNotifier ?? ValueNotifier(null), + imageProviderNotifier = imageProviderNotifier ?? ValueNotifier(null); // increase use of item void addUse() { @@ -463,27 +469,28 @@ class AssetCacheAll { final log = Logger('OrderedAssetCache'); - // return pdfDocument of asset it is lazy because it take some time to do it - Future getPdfDocument(int assetId) { - // if already opened, return it immediately + // pdfDocument notifier for rendering pdfs + ValueNotifier getPdfNotifier(int assetId) { final item = _items[assetId]; - if (item._pdfDocument != null) return Future.value(item._pdfDocument!); - - // if someone else is already opening this doc, return their future - final pending = _openingDocs[assetId]; - if (pending != null) return pending; - // otherwise start opening - final future = _openPdfDocument(item); - _openingDocs[assetId] = future; - - // when done, store the PdfDocument in the CacheItem and remove from _openingDocs - future.then((doc) { - item._pdfDocument = doc; - _openingDocs.remove(assetId); - }); + if (item._pdfDocument == null && item.value != null) { + // if no one is already opening this doc, return their future + if (_openingDocs[assetId] == null) { + final future = _openPdfDocument(item); + _openingDocs[assetId] = future; + + future.then((doc) { + item._pdfDocument = doc; + item.pdfDocumentNotifier.value = doc; // notify all widgets + _openingDocs.remove(assetId); + }); + } + } else if (item._pdfDocument != null) { + // if already opened, return it immediately + item.pdfDocumentNotifier.value = item._pdfDocument; + } - return future; + return item.pdfDocumentNotifier; } // open pdf document diff --git a/lib/components/canvas/image/pdf_editor_image.dart b/lib/components/canvas/image/pdf_editor_image.dart index 1b76d34f58..ea11a1b946 100644 --- a/lib/components/canvas/image/pdf_editor_image.dart +++ b/lib/components/canvas/image/pdf_editor_image.dart @@ -10,7 +10,7 @@ class PdfEditorImage extends EditorImage { /// that the pdf will be loaded from. final File? pdfFile; - final _pdfDocument = ValueNotifier(null); +// final _pdfDocument = ValueNotifier(null); static final log = Logger('PdfEditorImage'); @@ -157,7 +157,7 @@ class PdfEditorImage extends EditorImage { dstRect = dstRect.topLeft & dstSize; } - _pdfDocument.value ??= await assetCacheAll.getPdfDocument(assetId); +// _pdfDocument.value ??= await assetCacheAll.getPdfDocument(assetId); // _pdfDocument.value ??= pdfFile != null // ? await PdfDocument.openFile(pdfFile!.path) // : await PdfDocument.openData(pdfBytes!); @@ -171,19 +171,6 @@ class PdfEditorImage extends EditorImage { @override Future precache(BuildContext context) async { - if (_pdfDocument.value != null) return; - - final completer = Completer(); - - void onDocumentSet() { - if (_pdfDocument.value == null) return; - if (completer.isCompleted) return; - completer.complete(); - _pdfDocument.removeListener(onDocumentSet); - } - - _pdfDocument.addListener(onDocumentSet); - return completer.future; } @override @@ -193,8 +180,9 @@ class PdfEditorImage extends EditorImage { required bool isBackground, required bool invert, }) { + final pdfNotifier = assetCacheAll.getPdfNotifier(assetId); // value of pdfDocument return ValueListenableBuilder( - valueListenable: _pdfDocument, + valueListenable: pdfNotifier, builder: (context, pdfDocument, child) { if (pdfDocument == null) { return SizedBox.fromSize(size: srcRect.size); From eea7dbb4e90516ce43a895a9e3a93e1c5f3270d4 Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 8 Oct 2025 17:55:26 +0200 Subject: [PATCH 25/35] fixed file extension to contain '.' --- lib/components/canvas/image/pdf_editor_image.dart | 4 ++-- lib/components/canvas/image/svg_editor_image.dart | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/components/canvas/image/pdf_editor_image.dart b/lib/components/canvas/image/pdf_editor_image.dart index ea11a1b946..d82ada0fbe 100644 --- a/lib/components/canvas/image/pdf_editor_image.dart +++ b/lib/components/canvas/image/pdf_editor_image.dart @@ -61,7 +61,7 @@ class PdfEditorImage extends EditorImage { pdfFile = FileManager.getFile('$sbnPath${Editor.extension}.$assetIndexJson'); assetIndex = assetCacheAll.addSync( - pdfFile,'pdf', + pdfFile,'.pdf', json.containsKey('ainf') ? json['ainf'] : null, json.containsKey('aph') ? json['aph'].toInt() : null, json.containsKey('afs') ? json['afs'] : null, @@ -71,7 +71,7 @@ class PdfEditorImage extends EditorImage { pdfBytes = inlineAssets[assetIndexJson]; final tempFile=assetCacheAll.createRuntimeFile('.pdf',pdfBytes); // store to file assetIndex = assetCacheAll.addSync( - tempFile,'pdf', + tempFile,'.pdf', json.containsKey('ainf') ? json['ainf'] : null, json.containsKey('aph') ? json['aph'].toInt() : null, json.containsKey('afs') ? json['afs'] : null, diff --git a/lib/components/canvas/image/svg_editor_image.dart b/lib/components/canvas/image/svg_editor_image.dart index c924d028d3..0d6bba2459 100644 --- a/lib/components/canvas/image/svg_editor_image.dart +++ b/lib/components/canvas/image/svg_editor_image.dart @@ -61,7 +61,7 @@ class SvgEditorImage extends EditorImage { } if (svgFile != null) { assetIndex = assetCacheAll.addSync( - svgFile,'svg', + svgFile,'.svg', json.containsKey('ainf') ? json['ainf'] : null, json.containsKey('aph') ? json['aph'].toInt() : null, json.containsKey('afs') ? json['afs'] : null, From 7311b80b872d551f61d4ee43f579f6ca658bf7f0 Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 8 Oct 2025 17:55:58 +0200 Subject: [PATCH 26/35] assetCache - added AssetType for better recognition of asset in cache --- lib/components/canvas/_asset_cache.dart | 128 ++++++++++++++++++++---- 1 file changed, 109 insertions(+), 19 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index 87f7e230f8..46ccd13e73 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -313,13 +313,22 @@ class PreviewResult { PreviewResult(this.previewHash, this.fileSize, this.fileInfo); } + +// types of assets in chage +enum AssetType { + image, // png, jpg, ... + pdf, // pdf + svg, // svg + unknown, +} + // object in cache class CacheItem { Object? value; // value can change because file can be renamed + final AssetType assetType; // type of asset int? previewHash; // quick hash (from first 100KB bytes) int? hash; // hash can be calculated later String? fileInfo; // file information - /Info of pdf is implemented now - final ValueNotifier imageProviderNotifier; // image provider for png, svg as value listener @@ -336,7 +345,7 @@ class CacheItem { int assetIdOnSave=-1; // id of asset when the note is save (some assets can be skipped) CacheItem( - this.value, { + this.value,{ this.hash, this.filePath, this.previewHash, @@ -346,9 +355,29 @@ class CacheItem { this.bytes, ValueNotifier? pdfDocumentNotifier, ValueNotifier? imageProviderNotifier, - }) : pdfDocumentNotifier = pdfDocumentNotifier ?? ValueNotifier(null), + }) : assetType = _detectTypeFromExtension(fileExt), + pdfDocumentNotifier = pdfDocumentNotifier ?? ValueNotifier(null), imageProviderNotifier = imageProviderNotifier ?? ValueNotifier(null); + bool get isImage => assetType == AssetType.image; + bool get isPdf => assetType == AssetType.pdf; + bool get isSvg => assetType == AssetType.svg; + + // detect asset type according to file extension + static AssetType _detectTypeFromExtension(String? extension) { + if (extension == null) return AssetType.unknown; + + final ext = extension.toLowerCase(); + if (['.jpg', '.jpeg', '.png', '.gif', '.bmp'].contains(ext)) { + return AssetType.image; + } else if (ext == '.pdf') { + return AssetType.pdf; + } else if (ext == '.svg') { + return AssetType.svg; + } + return AssetType.unknown; + } + // increase use of item void addUse() { _refCount++; @@ -432,13 +461,44 @@ class CacheItem { } // move asset file to temporary file to avoid it is overwriten during note save - void moveAssetToTemporaryFile() { + Future moveAssetToTemporaryFile() async { final dir = FileManager.supportDirectory; String newPath = '${dir.path}/TmPmP_${RandomFileName.generateRandomFileName(fileExt != null ? fileExt! : 'tmp')}'; // update file path - value=(value as File).renameSync(newPath); // rename asset + //value=(value as File).renameSync(newPath); // rename asset + value=await safeMoveFile((value as File),newPath); // rename asset filePath=newPath; } + // function used to rename assets + Future safeMoveFile(File source, String newPath) async { + try { + return await source.rename(newPath); + } on FileSystemException catch (e) { + if (e.osError?.errorCode == 18) { + // Cross-device link, must copy by hand + final newFile = await source.copy(newPath); + await source.delete(); + return newFile; + } + rethrow; + } + } + + void safeMoveFileSync(File source, File destination) { + try { + source.renameSync(destination.path); + } on FileSystemException catch (e) { + if (e.osError?.errorCode == 18) { + // Cross-device link – použijeme copySync a deleteSync + source.copySync(destination.path); + source.deleteSync(); + } else { + rethrow; // zachová původní stacktrace + } + } + } + + // @override @@ -472,7 +532,9 @@ class AssetCacheAll { // pdfDocument notifier for rendering pdfs ValueNotifier getPdfNotifier(int assetId) { final item = _items[assetId]; - + if (item.assetType !=AssetType.pdf){ + throw('getPdfNotified for non pdf asset'); + } if (item._pdfDocument == null && item.value != null) { // if no one is already opening this doc, return their future if (_openingDocs[assetId] == null) { @@ -504,12 +566,34 @@ class AssetCacheAll { } } + // removes image provider of file from image Cache. + // important to call this when assets are renamed + Future clearPdfDocumentNotifier(int assetId) async { + // return cached provider if already available + final item = _items[assetId]; + if (item.assetType !=AssetType.pdf){ + throw('clearPdfDocumentNotifier for non pdf asset'); + } + if (item.pdfDocumentNotifier.value == null) + return; + if (item.value is File) { + item._pdfDocument = null; + _openingDocs.remove(assetId); // clear indicator + item.pdfDocumentNotifier.value = null; // set provider to null to inform widgets to reload + } + return; + } + + // removes image provider of file from image Cache. // important to call this when assets are renamed Future clearImageProvider(int assetId) async { - // return cached provider if already available + // clear image provider if already available final item = _items[assetId]; + if (item.assetType !=AssetType.image){ + throw('clearImageProvider for non image asset'); + } if (item.imageProviderNotifier.value == null) return; if (item.value is File) { @@ -524,6 +608,9 @@ class AssetCacheAll { ValueNotifier getImageProviderNotifier(int assetId) { // return cached provider if already available final item = _items[assetId]; + if (item.assetType !=AssetType.image){ + throw('getImageProviderNotifier for non image asset'); + } if (item.imageProviderNotifier.value != null) return item.imageProviderNotifier; @@ -690,6 +777,7 @@ class AssetCacheAll { // always is used File! Future add(Object value) async { return await _mutex.protect(() async { + log.info("add mutex taken"); if (value is File) { // files are first compared by file path final path = value.path; @@ -812,6 +900,7 @@ class AssetCacheAll { // this routine should be called before note is saved. It renumbers assets according their usage Future renumberBeforeSave(String noteFilePath) async{ await _mutex.protect(() async { + log.info("renumber mutex taken"); int currentId = -1; for (int i = 0; i < _items.length; i++) { final item = _items[i]; @@ -821,19 +910,19 @@ class AssetCacheAll { item.assetIdOnSave = currentId; if ((item.filePath) != newPath) { // move asset to correct file name - final bool isImage = item.imageProviderNotifier !=null; - final bool isPdf = item._pdfDocument !=null; - if (isImage) { + if (item.isImage) { await clearImageProvider(i); // remove imageProvider from cache - } else if (isPdf){ - + } else if (item.isPdf){ } - - item.value = - (item.value as File).renameSync(newPath); // rename asset + item.value = await item.safeMoveFile((item.value as File), newPath); // rename asset file +// item.value = (item.value as File).renameSync(newPath); // rename asset item.filePath = newPath; - item.invalidateImageProvider(); // invalidate image provider so new one is allocated - getImageProviderNotifier(i); // allocate new image provider to enable rendering + if (item.isImage) { + item.invalidateImageProvider(); // invalidate image provider so new one is allocated + getImageProviderNotifier(i); // allocate new image provider to enable rendering + } else if (item.isPdf){ + clearPdfDocumentNotifier(i); + } } } else { @@ -844,7 +933,7 @@ class AssetCacheAll { // file path of asset is the same as note path - asset file can be overwritten, // move it to the safe location await clearImageProvider(i); // remove imageProvider from cache - item.moveAssetToTemporaryFile(); // move asset to different file + await item.moveAssetToTemporaryFile(); // move asset to different file item.invalidateImageProvider(); // invalidate image provider so new one is allocated getImageProviderNotifier(i); // allocate new image provider to enable rendering } @@ -935,13 +1024,14 @@ class AssetCacheAll { // create temporary file from bytes when inline bytes are read File createRuntimeFile(String ext, Uint8List bytes) { - final dir = Directory.systemTemp; // Použití systémového temp adresáře + final dir = Directory.systemTemp; // use system temp // final dir = await getApplicationSupportDirectory(); final file = File('${dir.path}/TmPmP_${RandomFileName.generateRandomFileName(ext)}'); file.writeAsBytesSync(bytes, flush: true); return file; } + void dispose() { _items.clear(); _aliasMap.clear(); From 6db2918a2c48f1a0166772d0e5f623452f5340b0 Mon Sep 17 00:00:00 2001 From: QubaB Date: Sun, 12 Oct 2025 22:56:21 +0200 Subject: [PATCH 27/35] pdfrx version 2.1.16 or greater do not work on Windows and Saber crash on PdfDocument.OpenFile --- pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index 643cd7ed96..402f8f6f1f 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -144,7 +144,7 @@ dependencies: flutter_web_auth_2: ^4.0.0 - pdfrx: ^2.1.3 + pdfrx: 2.1.15 path: ^1.9.1 stow: ^0.5.0 From c361877339c84a7591a4084003786be52a26882b Mon Sep 17 00:00:00 2001 From: QubaB Date: Mon, 13 Oct 2025 12:08:47 +0200 Subject: [PATCH 28/35] added fixFileNameDelimiters and getTmpAssetDir --- lib/data/file_manager/file_manager.dart | 41 ++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/lib/data/file_manager/file_manager.dart b/lib/data/file_manager/file_manager.dart index a34c8e9ee8..3c33ab0d64 100644 --- a/lib/data/file_manager/file_manager.dart +++ b/lib/data/file_manager/file_manager.dart @@ -58,7 +58,7 @@ class FileManager { stows.customDataDir.value ?? await getDefaultDocumentsDirectory(); static Future getDefaultDocumentsDirectory() async => - '${(await getApplicationDocumentsDirectory()).path}/$appRootDirectoryPrefix'; + '${(await getApplicationDocumentsDirectory()).path}${Platform.pathSeparator}$appRootDirectoryPrefix'; static Future migrateDataDir() async { final oldDir = Directory(documentsDirectory); @@ -121,6 +121,45 @@ class FileManager { } } + // fix '\' and '/' according to OS to have all the same + static String fixFileNameDelimiters(String filePath){ + if (Platform.pathSeparator == '\\'){ + return filePath.replaceAll('/', '\\'); + } + return filePath.replaceAll('\\', '/'); + } + + + /// Creates a hidden folder in DocumentsDirectory to store temporary assets + /// it is usefully to store temporary files here, because files can be simply renamed when assets change - no need to copy them + static Future getTmpAssetDir() async { + try{ + final baseDir = Directory(documentsDirectory); + + // Define a hidden subfolder (dot prefix hides it on Unix systems) + final hiddenDir = Directory('${baseDir.path}${Platform.pathSeparator}.tmpAssets'); + + // Create the folder if it doesn’t exist + if (!await hiddenDir.exists()) { + await hiddenDir.create(recursive: true); + + // On Windows, add the "hidden" file attribute + if (Platform.isWindows) { + try { + await Process.run('attrib', ['+h', hiddenDir.path]); + } catch (e) { + log.info('Failed to set hidden attribute: $e'); + } + } + } + return hiddenDir; + } + on FileSystemException catch (e) { + log.info('getTmpAssetDir $e'); + rethrow; + } + } + @visibleForTesting static Future watchRootDirectory() async { Directory rootDir = Directory(documentsDirectory); From f8b67813ed82c76ec59e3f2f9e52cefe796870b6 Mon Sep 17 00:00:00 2001 From: QubaB Date: Mon, 13 Oct 2025 15:30:30 +0200 Subject: [PATCH 29/35] all temporary assets are stored to .tmpAssets directory in Document folder. It allows fast rename of file during saving note and skipping unused assets_test at the end of Editor is temporary directory deleted. to compare correctly filename paths fixFileNameDelimiters is called. --- lib/components/canvas/_asset_cache.dart | 112 ++++++++++++++++++++---- lib/data/file_manager/file_manager.dart | 8 +- lib/pages/editor/editor.dart | 4 +- 3 files changed, 100 insertions(+), 24 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index 46ccd13e73..436a4ce617 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -460,27 +460,57 @@ class CacheItem { imageProviderNotifier.value = null; // will be recreated on next access } - // move asset file to temporary file to avoid it is overwriten during note save + // move asset file to temporary file to avoid it is overwritten during note save Future moveAssetToTemporaryFile() async { - final dir = FileManager.supportDirectory; - String newPath = '${dir.path}/TmPmP_${RandomFileName.generateRandomFileName(fileExt != null ? fileExt! : 'tmp')}'; // update file path + Directory? dir; + try { + dir = await FileManager.getTmpAssetDir(); + } catch (e) { + dir = FileManager.supportDirectory; + } + String newPath = FileManager.fixFileNameDelimiters('${dir.path}${Platform.pathSeparator}TmPmP_${RandomFileName.generateRandomFileName(fileExt != null ? fileExt! : 'tmp')}'); // update file path //value=(value as File).renameSync(newPath); // rename asset value=await safeMoveFile((value as File),newPath); // rename asset filePath=newPath; } + // copy asset file to temporary file to avoid it is overwritten during note save + Future copyAssetToTemporaryFile() async { + Directory? dir; + try { + dir = await FileManager.getTmpAssetDir(); + } catch (e) { + dir = FileManager.supportDirectory; + } + String newPath = FileManager.fixFileNameDelimiters('${dir.path}${Platform.pathSeparator}TmPmP_${RandomFileName.generateRandomFileName(fileExt != null ? fileExt! : 'tmp')}'); // update file path + value=await (value as File).copy(newPath); // copy asset to new file + filePath=newPath; + } + // function used to rename assets Future safeMoveFile(File source, String newPath) async { + // first test if destination directory exists + final String parentDirectory = newPath.substring(0, newPath.lastIndexOf(Platform.pathSeparator)); + await Directory(parentDirectory).create(recursive: true); try { return await source.rename(newPath); } on FileSystemException catch (e) { - if (e.osError?.errorCode == 18) { + // Cross-device link (Android/Linux = 18, Windows = 17 or 32) + final code = e.osError?.errorCode ?? -1; + if (code == 18 || code == 17 || code == 32) { // Cross-device link, must copy by hand final newFile = await source.copy(newPath); - await source.delete(); + try { + await source.delete(); + } on FileSystemException catch (e) { + // we at least copied file so continue + final code = e.osError?.errorCode ?? -1; + } return newFile; } - rethrow; + else { + rethrow; + } } } @@ -488,12 +518,13 @@ class CacheItem { try { source.renameSync(destination.path); } on FileSystemException catch (e) { - if (e.osError?.errorCode == 18) { - // Cross-device link – použijeme copySync a deleteSync + final code = e.osError?.errorCode ?? -1; + if (code == 18 || code == 17 || code == 32) { + // Cross-device link – use copySync and deleteSync source.copySync(destination.path); source.deleteSync(); } else { - rethrow; // zachová původní stacktrace + rethrow; } } } @@ -513,7 +544,7 @@ class CacheItem { class AssetCacheAll { final List _items = []; final Map _aliasMap = - {}; // duplicit indices point to first indice - updated in finalize + {}; // duplicite indices point to first indice - updated in finalize final Map _previewHashIndex = {}; // Map from previewHash → first index in _items @@ -579,6 +610,7 @@ class AssetCacheAll { if (item.value is File) { item._pdfDocument = null; _openingDocs.remove(assetId); // clear indicator + item.pdfDocumentNotifier.value!.dispose(); // free pdf document item.pdfDocumentNotifier.value = null; // set provider to null to inform widgets to reload } return; @@ -587,7 +619,8 @@ class AssetCacheAll { // removes image provider of file from image Cache. - // important to call this when assets are renamed + // important to call this when assets are renamed otherwise image cache will provide cached image of file + // instead of new image. Future clearImageProvider(int assetId) async { // clear image provider if already available final item = _items[assetId]; @@ -775,7 +808,8 @@ class AssetCacheAll { // async add cache is used from Editor, when adding asset using file picker // always is used File! - Future add(Object value) async { +// by default is file copied to new file before adding to assets + Future add(Object value,{bool copyFromSource = true }) async { return await _mutex.protect(() async { log.info("add mutex taken"); if (value is File) { @@ -833,6 +867,11 @@ class AssetCacheAll { _items[existingHashIndex].addUse(); return existingHashIndex; } + // file should be added to assets + if (copyFromSource){ + // copy original file + await newItem.copyAssetToTemporaryFile(); + } _items.add(newItem); final index = _items.length - 1; _previewHashIndex[previewResult.previewHash] = @@ -913,6 +952,7 @@ class AssetCacheAll { if (item.isImage) { await clearImageProvider(i); // remove imageProvider from cache } else if (item.isPdf){ + clearPdfDocumentNotifier(i); } item.value = await item.safeMoveFile((item.value as File), newPath); // rename asset file // item.value = (item.value as File).renameSync(newPath); // rename asset @@ -921,7 +961,7 @@ class AssetCacheAll { item.invalidateImageProvider(); // invalidate image provider so new one is allocated getImageProviderNotifier(i); // allocate new image provider to enable rendering } else if (item.isPdf){ - clearPdfDocumentNotifier(i); + // nothing to do with pdf. It's document is created automatically on redraw of widget } } } @@ -932,10 +972,18 @@ class AssetCacheAll { if ((item.filePath)!.startsWith(noteFilePath)) { // file path of asset is the same as note path - asset file can be overwritten, // move it to the safe location - await clearImageProvider(i); // remove imageProvider from cache + if (item.isImage) { + await clearImageProvider(i); // remove imageProvider from cache + } else if (item.isPdf){ + clearPdfDocumentNotifier(i); + } await item.moveAssetToTemporaryFile(); // move asset to different file - item.invalidateImageProvider(); // invalidate image provider so new one is allocated - getImageProviderNotifier(i); // allocate new image provider to enable rendering + if (item.isImage) { + item.invalidateImageProvider(); // invalidate image provider so new one is allocated + getImageProviderNotifier(i); // allocate new image provider to enable rendering + } else if (item.isPdf){ + // nothing to do with pdf. It's document is created automatically on redraw of widget + } } } } @@ -1026,17 +1074,45 @@ class AssetCacheAll { File createRuntimeFile(String ext, Uint8List bytes) { final dir = Directory.systemTemp; // use system temp // final dir = await getApplicationSupportDirectory(); - final file = File('${dir.path}/TmPmP_${RandomFileName.generateRandomFileName(ext)}'); + final file = File('${dir.path}${Platform.pathSeparator}TmPmP_${RandomFileName.generateRandomFileName(ext)}'); file.writeAsBytesSync(bytes, flush: true); return file; } - void dispose() { _items.clear(); _aliasMap.clear(); _openingDocs.clear(); _previewHashIndex.clear(); + _cleanupTempFiles(); + } + + // remove all temporary assets which were not used + Future _cleanupTempFiles() async { + // List all entries in the temporary directory + try { + final dir = await FileManager.getTmpAssetDir(); + await for (final entity in dir.list(recursive: false)) { + // Check if it's a file (not a directory) + if (entity is File) { + final fileName = entity.uri.pathSegments.last; + + // Check if the name starts with 'TmPmP' + if (fileName.startsWith('TmPmP')) { + try { + await entity.delete(); + log.info('AssetCacheAll.dispose - Deleted: $fileName'); + } catch (e) { + log.info('AssetCacheAll.dispose - failed to deleted: $fileName'); + } + } + } + } + await dir.delete(recursive: true); // delete also temporary directory so it does not show in file lists + } catch (e) { + log.info('Error deleting assetCacheAll temp files: $e'); + } + } @override diff --git a/lib/data/file_manager/file_manager.dart b/lib/data/file_manager/file_manager.dart index 3c33ab0d64..cc732d0c15 100644 --- a/lib/data/file_manager/file_manager.dart +++ b/lib/data/file_manager/file_manager.dart @@ -224,22 +224,22 @@ class FileManager { static File getFile(String filePath) { if (shouldUseRawFilePath) { - return File(filePath); + return File(fixFileNameDelimiters(filePath)); } else { assert(filePath.startsWith('/'), 'Expected filePath to start with a slash, got $filePath'); - return File(documentsDirectory + filePath); + return File(fixFileNameDelimiters(documentsDirectory + filePath)); } } // return file path (add document directory if needed) static String getFilePath(String filePath) { if (shouldUseRawFilePath) { - return filePath; + return fixFileNameDelimiters(filePath); } else { assert(filePath.startsWith('/'), 'Expected filePath to start with a slash, got $filePath'); - return '$documentsDirectory$filePath'; + return fixFileNameDelimiters('$documentsDirectory$filePath'); } } diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index ce0b5f89ac..b0ef299796 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -927,7 +927,7 @@ class EditorState extends State { final filePath = coreInfo.filePath + Editor.extension; final Uint8List bson; coreInfo.assetCacheAll.allowRemovingAssets = false; - final fullPath = FileManager.getFilePath(filePath); // add full path of note + final fullPath = FileManager.fixFileNameDelimiters(FileManager.getFilePath(filePath)); // add full path of note and fix \ or / according to OS await coreInfo.assetCacheAll.renumberBeforeSave(fullPath); // renumber all assets try { // go through all pages and prepare Json of each page. @@ -1236,7 +1236,7 @@ class EditorState extends State { final emptyPage = coreInfo.pages.removeLast(); assert(emptyPage.isEmpty); - final raster = Printing.raster( + final raster = Printing.raster( // get page size of pdf (raster it in very low quality) pdfBytes, dpi: 1, ); From b9c1586b247c036464f50af3ecdea1dd7ceaf7a1 Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 15 Oct 2025 09:59:53 +0200 Subject: [PATCH 30/35] rewriten importPdfFromFilePath it does not use Printing.raster to determine number of pdf pages and their size. Instead it waits for opening pdfDocument. Printing.raster caused Saber crash on windows. This method is fast. --- lib/pages/editor/editor.dart | 75 ++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/lib/pages/editor/editor.dart b/lib/pages/editor/editor.dart index b0ef299796..afe92a01ca 100644 --- a/lib/pages/editor/editor.dart +++ b/lib/pages/editor/editor.dart @@ -11,6 +11,7 @@ import 'package:flutter/services.dart'; import 'package:flutter_quill/flutter_quill.dart' as flutter_quill; import 'package:keybinder/keybinder.dart'; import 'package:logging/logging.dart'; +import 'package:pdfrx/pdfrx.dart'; import 'package:printing/printing.dart'; import 'package:saber/components/canvas/_stroke.dart'; import 'package:saber/components/canvas/canvas.dart'; @@ -1223,46 +1224,62 @@ class EditorState extends State { Future importPdfFromFilePath(String path) async { final pdfFile = File(path); - final Uint8List pdfBytes; - try { - pdfBytes = await pdfFile.readAsBytes(); - } catch (e) { - log.severe('Failed to read file when importing $path: $e', e); + + // Add pdf to cache → returns assetId + int? assetIndex = await coreInfo.assetCacheAll.add(pdfFile); + if (assetIndex == null) { + log.severe('Failed to add PDF to cache'); return false; } - int? assetIndex = await coreInfo.assetCacheAll.add(pdfFile); // add pdf to cache + // Wait for PdfDocument to be ready to know page size and number of pages + final pdfNotifier = coreInfo.assetCacheAll.getPdfNotifier(assetIndex); + // Wait until the document is loaded (non-null value) + PdfDocument? pdfDocument; + pdfDocument = pdfNotifier.value; + if (pdfDocument == null) { + final completer = Completer(); + late VoidCallback listener; + listener = () { + if (pdfNotifier.value != null) { + completer.complete(pdfNotifier.value!); + pdfNotifier.removeListener(listener); + } + }; + pdfNotifier.addListener(listener); + pdfDocument = await completer.future; + } + + // At this point, pdfDocument is guaranteed to be loaded + log.info('PDF loaded with ${pdfDocument.pages.length} pages'); + // Remove the temporary page final emptyPage = coreInfo.pages.removeLast(); assert(emptyPage.isEmpty); - final raster = Printing.raster( // get page size of pdf (raster it in very low quality) - pdfBytes, - dpi: 1, - ); - - int currentPdfPage = -1; - await for (final pdfPage in raster) { - ++currentPdfPage; - assert(currentPdfPage >= 0); + // Create pages from the actual PDF document + for (int i = 0; i < pdfDocument.pages.length; i++) { + final pdfPage = pdfDocument.pages[i]; // PdfPage object + final naturalSize = Size(pdfPage.width.toDouble(), pdfPage.height.toDouble()); - // resize to [defaultWidth] to keep pen sizes consistent + // Scale to your EditorPage default width final pageSize = Size( EditorPage.defaultWidth, - EditorPage.defaultWidth * pdfPage.height / pdfPage.width, + EditorPage.defaultWidth * naturalSize.height / naturalSize.width, ); - final page = EditorPage( + final editorPage = EditorPage( width: pageSize.width, height: pageSize.height, ); - page.backgroundImage = PdfEditorImage( + + editorPage.backgroundImage = PdfEditorImage( id: coreInfo.nextImageId++, pdfFile: pdfFile, - pdfPage: currentPdfPage, + pdfPage: i, pageIndex: coreInfo.pages.length, pageSize: pageSize, - naturalSize: Size(pdfPage.width.toDouble(), pdfPage.height.toDouble()), + naturalSize: naturalSize, onMoveImage: onMoveImage, onDeleteImage: onDeleteImage, onMiscChange: autosaveAfterDelay, @@ -1270,31 +1287,31 @@ class EditorState extends State { assetCacheAll: coreInfo.assetCacheAll, assetId: assetIndex, ); - coreInfo.assetCacheAll.addUse(assetIndex); // inform that asset is uded more times - coreInfo.pages.add(page); + + coreInfo.assetCacheAll.addUse(assetIndex); + coreInfo.pages.add(editorPage); + history.recordChange(EditorHistoryItem( type: EditorHistoryItemType.insertPage, pageIndex: coreInfo.pages.length - 1, strokes: const [], images: const [], - page: page, + page: editorPage, )); - if (currentPdfPage == 0) { - // update ui after we've rastered the first page - // so that the user has some indication that the import is working - setState(() {}); + if (i == 0) { + setState(() {}); // Update early to show progress } } coreInfo.pages.add(emptyPage); setState(() {}); - autosaveAfterDelay(); return true; } + Future paste() async { /// Maps image formats to their file extension. const Map formats = { From 2280d079d578a85be846a93b08e6efdb3e476f68 Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 15 Oct 2025 12:58:09 +0200 Subject: [PATCH 31/35] fixed saving assets (correct renaming of assets if some is deleted). fixed adding assets using addSync when order of assets is not correct. --- lib/components/canvas/_asset_cache.dart | 135 +++++++++++++++++++----- 1 file changed, 108 insertions(+), 27 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index 436a4ce617..1d94364bef 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'dart:isolate'; import 'dart:math'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; @@ -359,6 +360,17 @@ class CacheItem { pdfDocumentNotifier = pdfDocumentNotifier ?? ValueNotifier(null), imageProviderNotifier = imageProviderNotifier ?? ValueNotifier(null); + /// Placeholder constructor + CacheItem.placeholder() + : value = null, + fileExt = null, + fileSize = null, + assetType = AssetType.unknown, + filePath = null, + pdfDocumentNotifier = ValueNotifier(null), + imageProviderNotifier = ValueNotifier(null); + + bool get isImage => assetType == AssetType.image; bool get isPdf => assetType == AssetType.pdf; bool get isSvg => assetType == AssetType.svg; @@ -378,6 +390,17 @@ class CacheItem { return AssetType.unknown; } + void dispose(){ + if (isPdf) { + if (pdfDocumentNotifier.value == null) + return; + if (value is File) { + _pdfDocument!.dispose(); + pdfDocumentNotifier.value = null; // set provider to null to inform widgets to reload + } + } + } + // increase use of item void addUse() { _refCount++; @@ -566,30 +589,42 @@ class AssetCacheAll { if (item.assetType !=AssetType.pdf){ throw('getPdfNotified for non pdf asset'); } + log.info('getPdfNotifiert ${assetId}'); if (item._pdfDocument == null && item.value != null) { // if no one is already opening this doc, return their future if (_openingDocs[assetId] == null) { - final future = _openPdfDocument(item); - _openingDocs[assetId] = future; - - future.then((doc) { + _openingDocs[assetId] = _openPdfDocument(item) + .then((doc) { + log.info('_pdfDocument read for $assetId'); item._pdfDocument = doc; - item.pdfDocumentNotifier.value = doc; // notify all widgets + item.pdfDocumentNotifier.value = doc; // notify widgets _openingDocs.remove(assetId); + return doc; + }).catchError((e, st) { + log.severe('Error opening PDF $assetId: $e\n$st'); + _openingDocs.remove(assetId); + throw e; // ❌ místo `return null` }); } } else if (item._pdfDocument != null) { // if already opened, return it immediately - item.pdfDocumentNotifier.value = item._pdfDocument; + if (item.pdfDocumentNotifier.value != item._pdfDocument) { + item.pdfDocumentNotifier.value = item._pdfDocument; + } } - + log.info('_pdfDocument returning'); return item.pdfDocumentNotifier; } // open pdf document Future _openPdfDocument(CacheItem item) async { if (item.filePath != null) { - return PdfDocument.openFile(item.filePath!); + log.info('open PdfDocument ${item.filePath!}'); +// final file = File(item.filePath!); +// item.bytes = await file.readAsBytes(); +// return PdfDocument.openData(item.bytes!); + log.info('${Isolate.current.debugName}'); + return await PdfDocument.openFile(item.filePath!); } else if (item.value is Uint8List) { return PdfDocument.openData(item.value as Uint8List); } else { @@ -741,6 +776,7 @@ class AssetCacheAll { // full hashes are established later int addSync(Object value, String extension, + int assetIdNote, // assets of note can be read not in order of asset number. String? fileInfo, int? previewHash, int? fileSize, @@ -794,9 +830,11 @@ class AssetCacheAll { fileInfo: previewResult.fileInfo, hash: hash) ..addUse(); // and add use - - _items.add(newItem); - final index = _items.length - 1; + while (_items.length <= assetIdNote) { + _items.add(CacheItem.placeholder()); // insert placeholder of asset, it will be filled later + } + _items[assetIdNote]=newItem; + final index = assetIdNote; _previewHashIndex[previewResult.previewHash] = index; // add to previously hashed return index; @@ -937,26 +975,52 @@ class AssetCacheAll { } // this routine should be called before note is saved. It renumbers assets according their usage + // if some asset is deleted (it is kept in undo/redo cache). When undo is used, asset is used again, but if note is saved meantime, + // it can occur situation that item[1] is again used, but item[2] has already asset file name *.1 and will be rewriten when item[1] is saved + // we must use 2 pass check of file names Future renumberBeforeSave(String noteFilePath) async{ await _mutex.protect(() async { log.info("renumber mutex taken"); + + // first assign asset number for files + log.info("find asset numbers"); int currentId = -1; for (int i = 0; i < _items.length; i++) { final item = _items[i]; + log.info("item $i"); if (item.refCount > 0) { currentId++; - final newPath = '$noteFilePath.$currentId'; // new asset file item.assetIdOnSave = currentId; - if ((item.filePath) != newPath) { - // move asset to correct file name + } + else { + // this item is not used. We should save it to temp directory + // in case of undo it can be used again + item.assetIdOnSave = -1; + } + } + + log.info("move unused assets to temporary files"); + for (int i = _items.length - 1; i >= 0; i--) { + final item = _items[i]; + log.info("item $i"); + if (item.refCount < 1) { + // this item is not used. We should save it to temp directory + // in case of undo it can be used again + log.info("item $i is not used as asset"); + if ((item.filePath)!.startsWith(noteFilePath)) { + log.info("item is from asset and must be saved to temporary directory"); + // file path of asset is the same as note path - asset file can be overwritten, + // move it to the safe location if (item.isImage) { await clearImageProvider(i); // remove imageProvider from cache } else if (item.isPdf){ clearPdfDocumentNotifier(i); } - item.value = await item.safeMoveFile((item.value as File), newPath); // rename asset file -// item.value = (item.value as File).renameSync(newPath); // rename asset - item.filePath = newPath; + try { + await item.moveAssetToTemporaryFile(); // move asset to different file + } catch (e){ + log.info('Error saving asset to temporary file'); + } if (item.isImage) { item.invalidateImageProvider(); // invalidate image provider so new one is allocated getImageProviderNotifier(i); // allocate new image provider to enable rendering @@ -965,19 +1029,27 @@ class AssetCacheAll { } } } - else { - // this item is not used. We should save it to temp directory - // in case of undo it can be used again - item.assetIdOnSave = -1; - if ((item.filePath)!.startsWith(noteFilePath)) { - // file path of asset is the same as note path - asset file can be overwritten, - // move it to the safe location + } + + + log.info("now assetId on Save are assigned and assets can be saved - it is safe to save them in reversed order"); + for (int i = _items.length - 1; i >= 0; i--) { + final item = _items[i]; + log.info("item $i"); + if (item.refCount > 0) { + currentId=item.assetIdOnSave; // assign asset number on save + final newPath = '$noteFilePath.$currentId'; // new asset file + log.info("item $i is used as asset $currentId. item file name ${item.filePath} and must be saved as $newPath"); + if ((item.filePath) != newPath) { + // move asset to correct file name if (item.isImage) { await clearImageProvider(i); // remove imageProvider from cache } else if (item.isPdf){ clearPdfDocumentNotifier(i); } - await item.moveAssetToTemporaryFile(); // move asset to different file + item.value = await item.safeMoveFile((item.value as File), newPath); // rename asset file +// item.value = (item.value as File).renameSync(newPath); // rename asset + item.filePath = newPath; if (item.isImage) { item.invalidateImageProvider(); // invalidate image provider so new one is allocated getImageProviderNotifier(i); // allocate new image provider to enable rendering @@ -986,6 +1058,11 @@ class AssetCacheAll { } } } + else { + // this item is not used. We should save it to temp directory + // in case of undo it can be used again + // but it was already moved to temporary file + } } return; }); @@ -1080,15 +1157,19 @@ class AssetCacheAll { } void dispose() { + for (CacheItem item in _items){ + item.dispose(); + } _items.clear(); + _cleanupCacheAll(); _aliasMap.clear(); _openingDocs.clear(); _previewHashIndex.clear(); - _cleanupTempFiles(); } // remove all temporary assets which were not used - Future _cleanupTempFiles() async { + Future _cleanupCacheAll() async { + // List all entries in the temporary directory try { final dir = await FileManager.getTmpAssetDir(); From e2f5a0fe265bd257b6e3f0c31fd668cd55da1915 Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 15 Oct 2025 12:59:10 +0200 Subject: [PATCH 32/35] addSync uses also assetIndex from Json to keep order of assets (background pdfs are read last) --- lib/components/canvas/image/pdf_editor_image.dart | 4 ++-- lib/components/canvas/image/png_editor_image.dart | 4 ++-- lib/components/canvas/image/svg_editor_image.dart | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/components/canvas/image/pdf_editor_image.dart b/lib/components/canvas/image/pdf_editor_image.dart index d82ada0fbe..28e78a47b0 100644 --- a/lib/components/canvas/image/pdf_editor_image.dart +++ b/lib/components/canvas/image/pdf_editor_image.dart @@ -61,7 +61,7 @@ class PdfEditorImage extends EditorImage { pdfFile = FileManager.getFile('$sbnPath${Editor.extension}.$assetIndexJson'); assetIndex = assetCacheAll.addSync( - pdfFile,'.pdf', + pdfFile,'.pdf',assetIndexJson, json.containsKey('ainf') ? json['ainf'] : null, json.containsKey('aph') ? json['aph'].toInt() : null, json.containsKey('afs') ? json['afs'] : null, @@ -71,7 +71,7 @@ class PdfEditorImage extends EditorImage { pdfBytes = inlineAssets[assetIndexJson]; final tempFile=assetCacheAll.createRuntimeFile('.pdf',pdfBytes); // store to file assetIndex = assetCacheAll.addSync( - tempFile,'.pdf', + tempFile,'.pdf',assetIndexJson, json.containsKey('ainf') ? json['ainf'] : null, json.containsKey('aph') ? json['aph'].toInt() : null, json.containsKey('afs') ? json['afs'] : null, diff --git a/lib/components/canvas/image/png_editor_image.dart b/lib/components/canvas/image/png_editor_image.dart index d9c082e680..9ae3049d93 100644 --- a/lib/components/canvas/image/png_editor_image.dart +++ b/lib/components/canvas/image/png_editor_image.dart @@ -87,7 +87,7 @@ class PngEditorImage extends EditorImage { // add to asset cache if (imageFile != null) { assetIndex = assetCacheAll.addSync( - imageFile,ext!, + imageFile,ext!,assetIndexJson!, json.containsKey('ainf') ? json['ainf'] : null, json.containsKey('aph') ? json['aph'].toInt() : null, json.containsKey('afs') ? json['afs'] : null, @@ -97,7 +97,7 @@ class PngEditorImage extends EditorImage { else { final tempFile=assetCacheAll.createRuntimeFile(ext!,bytes!); assetIndex = assetCacheAll.addSync( - tempFile,ext, + tempFile,ext,assetIndexJson!, json.containsKey('ainf') ? json['ainf'] : null, json.containsKey('aph') ? json['aph'].toInt() : null, json.containsKey('afs') ? json['afs'] : null, diff --git a/lib/components/canvas/image/svg_editor_image.dart b/lib/components/canvas/image/svg_editor_image.dart index 0d6bba2459..41552088b3 100644 --- a/lib/components/canvas/image/svg_editor_image.dart +++ b/lib/components/canvas/image/svg_editor_image.dart @@ -61,7 +61,7 @@ class SvgEditorImage extends EditorImage { } if (svgFile != null) { assetIndex = assetCacheAll.addSync( - svgFile,'.svg', + svgFile,'.svg',assetIndexJson!, json.containsKey('ainf') ? json['ainf'] : null, json.containsKey('aph') ? json['aph'].toInt() : null, json.containsKey('afs') ? json['afs'] : null, From 8d404a223f1d0964f511a3581290e6d91fd76b86 Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 15 Oct 2025 12:59:54 +0200 Subject: [PATCH 33/35] use of the newest pdrfx 2.2.1 --- pubspec.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index 402f8f6f1f..8594e41fea 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -144,7 +144,7 @@ dependencies: flutter_web_auth_2: ^4.0.0 - pdfrx: 2.1.15 + pdfrx: 2.2.1 path: ^1.9.1 stow: ^0.5.0 @@ -179,6 +179,8 @@ dev_dependencies: sentry_dart_plugin: ^3.1.1 dependency_overrides: + + # https://github.com/KasemJaffer/receive_sharing_intent/pull/333 receive_sharing_intent: git: From aa9c41038a5cd393a5ad3b7a593570ae64e06d14 Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 15 Oct 2025 13:05:27 +0200 Subject: [PATCH 34/35] removed test of isolate --- lib/components/canvas/_asset_cache.dart | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index 1d94364bef..c51d3fb59d 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -1,7 +1,6 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; -import 'dart:isolate'; import 'dart:math'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; @@ -623,7 +622,6 @@ class AssetCacheAll { // final file = File(item.filePath!); // item.bytes = await file.readAsBytes(); // return PdfDocument.openData(item.bytes!); - log.info('${Isolate.current.debugName}'); return await PdfDocument.openFile(item.filePath!); } else if (item.value is Uint8List) { return PdfDocument.openData(item.value as Uint8List); From cba200e17a52b82bfddb55721dec30c248532e67 Mon Sep 17 00:00:00 2001 From: QubaB Date: Wed, 15 Oct 2025 21:32:27 +0200 Subject: [PATCH 35/35] fileManager: removed supportDirectory. It is not used and blocks to use FileManager in isolate (when reading large notes on background). --- lib/components/canvas/_asset_cache.dart | 4 ++-- lib/data/file_manager/file_manager.dart | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/components/canvas/_asset_cache.dart b/lib/components/canvas/_asset_cache.dart index c51d3fb59d..dff716104a 100644 --- a/lib/components/canvas/_asset_cache.dart +++ b/lib/components/canvas/_asset_cache.dart @@ -488,7 +488,7 @@ class CacheItem { try { dir = await FileManager.getTmpAssetDir(); } catch (e) { - dir = FileManager.supportDirectory; + throw('Error getting temporary directory'); } String newPath = FileManager.fixFileNameDelimiters('${dir.path}${Platform.pathSeparator}TmPmP_${RandomFileName.generateRandomFileName(fileExt != null ? fileExt! : 'tmp')}'); // update file path //value=(value as File).renameSync(newPath); // rename asset @@ -502,7 +502,7 @@ class CacheItem { try { dir = await FileManager.getTmpAssetDir(); } catch (e) { - dir = FileManager.supportDirectory; + throw('Error getting temporary directory'); } String newPath = FileManager.fixFileNameDelimiters('${dir.path}${Platform.pathSeparator}TmPmP_${RandomFileName.generateRandomFileName(fileExt != null ? fileExt! : 'tmp')}'); // update file path value=await (value as File).copy(newPath); // copy asset to new file diff --git a/lib/data/file_manager/file_manager.dart b/lib/data/file_manager/file_manager.dart index cc732d0c15..4b303f6204 100644 --- a/lib/data/file_manager/file_manager.dart +++ b/lib/data/file_manager/file_manager.dart @@ -31,9 +31,6 @@ class FileManager { /// Realistically, this value never changes. static late String documentsDirectory; - // Support directory is for storing intermediate files (these files can be cleaned after note is saved) - static late Directory supportDirectory; - static final fileWriteStream = StreamController.broadcast(); // TODO(adil192): Implement or remove this @@ -49,7 +46,6 @@ class FileManager { }) async { FileManager.documentsDirectory = documentsDirectory ?? await getDocumentsDirectory(); - FileManager.supportDirectory = await getApplicationSupportDirectory(); if (shouldWatchRootDirectory) unawaited(watchRootDirectory()); }