diff --git a/.env b/.env index c5313cf..b27c3ed 100644 --- a/.env +++ b/.env @@ -5,3 +5,8 @@ PASSWORD=master66 JWT_SECRET=123456789 JWT_EXPIRES=1h PATH_FULL=true +LOG_MODE=both # console | file | both +LOG_FILE=scan.log # nome file log +LOG_DIR=public +LOG_VERBOSE=true + diff --git a/api_v1/scanner/elevation.js b/api_v1/scanner/elevation.js new file mode 100644 index 0000000..3cdab02 --- /dev/null +++ b/api_v1/scanner/elevation.js @@ -0,0 +1,22 @@ +// elevation.js +async function getElevation(lat, lon) { + console.log("OpenElevation request:", lat, lon); + + const url = `https://api.open-elevation.com/api/v1/lookup?locations=${lat},${lon}`; + + try { + const res = await fetch(url); + console.log("OpenElevation status:", res.status); + + const json = await res.json(); + console.log("OpenElevation response:", json); + + return json?.results?.[0]?.elevation ?? null; + + } catch (err) { + console.warn("Errore OpenElevation:", err); + return null; + } +} + +module.exports = { getElevation }; diff --git a/api_v1/scanner/indexStore.js b/api_v1/scanner/indexStore.js index e4795d6..7f00eac 100644 --- a/api_v1/scanner/indexStore.js +++ b/api_v1/scanner/indexStore.js @@ -8,35 +8,16 @@ const absIndexTmp = absIndexPath + '.tmp'; const PRETTY = process.env.INDEX_PRETTY === 'true'; -/** - * Carica l'indice strutturato: - * { - * user: { - * cartella: { - * id: { id, user, cartella, path, hash } - * } - * } - * } - */ async function loadPreviousIndex() { try { const raw = await fsp.readFile(absIndexPath, 'utf8'); const json = JSON.parse(raw); - - if (json && typeof json === 'object' && !Array.isArray(json)) { - return json; - } - - console.log("[INDEX] Formato vecchio → forzo rescan completo"); - return {}; + return (json && typeof json === 'object') ? json : {}; } catch { return {}; } } -/** - * Salva l'indice in modo atomico. - */ async function saveIndex(indexTree) { const dir = path.dirname(absIndexPath); await fsp.mkdir(dir, { recursive: true }); diff --git a/api_v1/scanner/indexStore.js.ok b/api_v1/scanner/indexStore.js.ok deleted file mode 100644 index e4795d6..0000000 --- a/api_v1/scanner/indexStore.js.ok +++ /dev/null @@ -1,52 +0,0 @@ -// scanner/indexStore.js -const path = require('path'); -const fsp = require('fs/promises'); -const { WEB_ROOT, INDEX_PATH } = require('../config'); - -const absIndexPath = path.resolve(__dirname, '..', '..', WEB_ROOT, INDEX_PATH); -const absIndexTmp = absIndexPath + '.tmp'; - -const PRETTY = process.env.INDEX_PRETTY === 'true'; - -/** - * Carica l'indice strutturato: - * { - * user: { - * cartella: { - * id: { id, user, cartella, path, hash } - * } - * } - * } - */ -async function loadPreviousIndex() { - try { - const raw = await fsp.readFile(absIndexPath, 'utf8'); - const json = JSON.parse(raw); - - if (json && typeof json === 'object' && !Array.isArray(json)) { - return json; - } - - console.log("[INDEX] Formato vecchio → forzo rescan completo"); - return {}; - } catch { - return {}; - } -} - -/** - * Salva l'indice in modo atomico. - */ -async function saveIndex(indexTree) { - const dir = path.dirname(absIndexPath); - await fsp.mkdir(dir, { recursive: true }); - - const data = PRETTY - ? JSON.stringify(indexTree, null, 2) - : JSON.stringify(indexTree); - - await fsp.writeFile(absIndexTmp, data, 'utf8'); - await fsp.rename(absIndexTmp, absIndexPath); -} - -module.exports = { loadPreviousIndex, saveIndex, absIndexPath }; diff --git a/api_v1/scanner/l.js b/api_v1/scanner/l.js deleted file mode 100644 index 0e1293a..0000000 --- a/api_v1/scanner/l.js +++ /dev/null @@ -1,73 +0,0 @@ -// tools/listFolderIds.js -const fs = require('fs'); -const fsp = require('fs/promises'); -const path = require('path'); -const readline = require('readline'); -const { WEB_ROOT, INDEX_PATH } = require('../config'); - -async function buildIdsListForFolder(userName, cartella) { - const indexPath = path.resolve(__dirname, '..', '..', WEB_ROOT, INDEX_PATH); - const idsIndex = []; - - console.log("\n[LIST] buildIdsListForFolder()"); - console.log("[LIST] user =", userName); - console.log("[LIST] cartella=", cartella); - console.log("[LIST] indexPath =", indexPath); - - if (!fs.existsSync(indexPath)) { - console.log("[LIST] index.json non trovato"); - return idsIndex; - } - - try { - const raw = await fsp.readFile(indexPath, 'utf8'); - const index = JSON.parse(raw); - - const userObj = index[userName]; - if (!userObj) { - console.log(`[LIST] Nessuna entry per user=${userName}`); - return idsIndex; - } - - const folder = userObj[cartella]; - if (!folder) { - console.log(`[LIST] Nessuna cartella "${cartella}" per user=${userName}`); - return idsIndex; - } - - const ids = Object.keys(folder).filter(k => k !== "_folderHash"); - idsIndex.push(...ids); - - console.log(`\n[LIST] ID trovati in ${userName}/${cartella}: ${idsIndex.length}`); - idsIndex.forEach(id => console.log(" -", id)); - console.log(""); - - return idsIndex; - - } catch (err) { - console.log("[LIST] Errore leggendo/parsing index.json:", err.message); - return idsIndex; - } -} - -async function main() { - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout - }); - - const ask = (q) => new Promise(res => rl.question(q, res)); - - try { - const user = await ask('User: '); - const cartella = await ask('Cartella: '); - - await buildIdsListForFolder(user.trim(), cartella.trim()); - } finally { - rl.close(); - } -} - -main().catch(err => { - console.error("Errore:", err); -}); diff --git a/api_v1/scanner/logger.js b/api_v1/scanner/logger.js new file mode 100644 index 0000000..1e8abde --- /dev/null +++ b/api_v1/scanner/logger.js @@ -0,0 +1,43 @@ +// scanner/logger.js +const fs = require('fs'); +const path = require('path'); + +const LOG_MODE = process.env.LOG_MODE || "console"; // console | file | both +const LOG_DIR = process.env.LOG_DIR || null; +const LOG_FILE = process.env.LOG_FILE || "scan.log"; + +let stream = null; + +function resolveLogPath() { + if (LOG_DIR) { + return path.resolve(__dirname, "..", "..", LOG_DIR, LOG_FILE); + } + return path.resolve(__dirname, "..", "..", LOG_FILE); +} + +if (LOG_MODE === "file" || LOG_MODE === "both") { + const logPath = resolveLogPath(); + + // crea la directory se non esiste + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + + stream = fs.createWriteStream(logPath, { flags: "a" }); +} + +function ts() { + return new Date().toISOString().replace("T", " ").split(".")[0]; +} + +function log(message) { + const line = `${ts()} ${message}\n`; + + if (LOG_MODE === "console" || LOG_MODE === "both") { + process.stdout.write(line); + } + + if (LOG_MODE === "file" || LOG_MODE === "both") { + stream.write(line); + } +} + +module.exports = { log }; diff --git a/api_v1/scanner/logger.js.ok b/api_v1/scanner/logger.js.ok new file mode 100644 index 0000000..f1ac187 --- /dev/null +++ b/api_v1/scanner/logger.js.ok @@ -0,0 +1,31 @@ +// scanner/logger.js +const fs = require('fs'); +const path = require('path'); + +const LOG_MODE = process.env.LOG_MODE || "console"; // console | file | both +const LOG_FILE = process.env.LOG_FILE || "scan.log"; + +let stream = null; + +if (LOG_MODE === "file" || LOG_MODE === "both") { + const logPath = path.resolve(__dirname, "..", LOG_FILE); + stream = fs.createWriteStream(logPath, { flags: "a" }); +} + +function ts() { + return new Date().toISOString().replace("T", " ").split(".")[0]; +} + +function log(message) { + const line = `${ts()} ${message}\n`; + + if (LOG_MODE === "console" || LOG_MODE === "both") { + process.stdout.write(line); + } + + if (LOG_MODE === "file" || LOG_MODE === "both") { + stream.write(line); + } +} + +module.exports = { log }; diff --git a/api_v1/scanner/orphanCleanup.js b/api_v1/scanner/orphanCleanup.js index abb8bbe..ca03a56 100644 --- a/api_v1/scanner/orphanCleanup.js +++ b/api_v1/scanner/orphanCleanup.js @@ -1,136 +1,69 @@ -// scanner/orphanCleanup.js -const fs = require('fs'); -const fsp = require('fs/promises'); -const path = require('path'); -const { WEB_ROOT, INDEX_PATH } = require('../config'); - -module.exports = function createCleanupFunctions(db) { - - async function buildIdsListForFolder(userName, cartella) { - const indexPath = path.resolve(__dirname, '..', '..', WEB_ROOT, INDEX_PATH); - const idsIndex = []; - - console.log("\n[ORPHAN] buildIdsListForFolder()"); - console.log("[ORPHAN] user =", userName); - console.log("[ORPHAN] cartella =", cartella); - console.log("[ORPHAN] indexPath =", indexPath); - - if (!fs.existsSync(indexPath)) { - console.log("[ORPHAN] index.json non trovato"); - return idsIndex; - } - - try { - const raw = await fsp.readFile(indexPath, 'utf8'); - const index = JSON.parse(raw); - - const userObj = index[userName]; - if (!userObj) return idsIndex; - - const folder = userObj[cartella]; - if (!folder) return idsIndex; - - const ids = Object.keys(folder).filter(k => k !== "_folderHash"); - idsIndex.push(...ids); - - console.log(`[ORPHAN] ID trovati in ${userName}/${cartella}: ${idsIndex.length}`); - idsIndex.forEach(id => console.log(" -", id)); - - return idsIndex; - - } catch (err) { - console.log("[ORPHAN] Errore leggendo index.json:", err.message); - return idsIndex; - } - } - - function removeIdFromList(idsIndex, id) { - const newList = idsIndex.filter(x => x !== id); - - if (newList.length !== idsIndex.length) { - console.log(`[ORPHAN] Rimosso ID dalla lista: ${id}`); - } else { - console.log(`[ORPHAN] ID NON presente nella lista: ${id}`); - } - - console.log(`[ORPHAN] ID rimanenti (${newList.length}):`); - newList.forEach(x => console.log(" -", x)); - console.log(""); - - return newList; - } - - async function deleteFromIndexById(id) { - const indexPath = path.resolve(__dirname, '..', '..', WEB_ROOT, INDEX_PATH); - - if (!fs.existsSync(indexPath)) return false; - - const raw = await fsp.readFile(indexPath, 'utf8'); - const index = JSON.parse(raw); - - let deleted = false; - - for (const user of Object.keys(index)) { - for (const cartella of Object.keys(index[user])) { - const folder = index[user][cartella]; - if (folder[id]) { - delete folder[id]; - deleted = true; - console.log(`✔ Eliminato ID ${id} da ${user}/${cartella}`); - } - } - } - - if (deleted) { - await fsp.writeFile(indexPath, JSON.stringify(index, null, 2)); - } - - return deleted; - } - - async function deleteThumbsById(id) { - const col = db.get('photos'); - const rec = col.find({ id }).value(); - - if (!rec) { - console.log(`[ORPHAN] Nessun record DB per ID ${id}`); - return false; - } - - const thumbs = [rec.thub1, rec.thub2].filter(Boolean); - let deleted = false; - - for (const t of thumbs) { - const abs = path.resolve(__dirname, '../public' + t); - if (fs.existsSync(abs)) { - await fsp.rm(abs, { force: true }); - console.log(`✔ Eliminato thumb: ${abs}`); - deleted = true; - } - } - - return deleted; - } - - function deleteFromDB(id) { - const col = db.get('photos'); - const exists = col.find({ id }).value(); - - if (exists) { - col.remove({ id }).write(); - console.log(`✔ Eliminato ID ${id} dal DB`); - return true; - } - - console.log(`[ORPHAN] ID ${id} non presente nel DB`); - return false; - } - - return { - buildIdsListForFolder, - removeIdFromList, - deleteFromIndexById, - deleteThumbsById, - deleteFromDB - }; -}; +// scanner/orphanCleanup.js +const fs = require('fs'); +const fsp = require('fs/promises'); +const path = require('path'); +const { WEB_ROOT, INDEX_PATH } = require('../config'); + +module.exports = function createCleanupFunctions(db) { + + async function buildIdsListForFolder(userName, cartella) { + const indexPath = path.resolve(__dirname, '..', '..', WEB_ROOT, INDEX_PATH); + const idsIndex = []; + + if (!fs.existsSync(indexPath)) return idsIndex; + + try { + const raw = await fsp.readFile(indexPath, 'utf8'); + const index = JSON.parse(raw); + + const folder = index?.[userName]?.[cartella]; + if (!folder) return idsIndex; + + return Object.keys(folder).filter(k => k !== "_folderHash"); + } catch { + return idsIndex; + } + } + + function removeIdFromList(idsIndex, id) { + return idsIndex.filter(x => x !== id); + } + + async function deleteThumbsById(id) { + const col = db.get('photos'); + const rec = col.find({ id }).value(); + if (!rec) return false; + + const thumbs = [rec.thub1, rec.thub2].filter(Boolean); + let deleted = false; + + for (const t of thumbs) { + const abs = path.resolve(__dirname, '../public' + t); + if (fs.existsSync(abs)) { + await fsp.rm(abs, { force: true }); + console.log(` 🔴 Thumb eliminato: ${abs}`); + deleted = true; + } + } + return deleted; + } + + function deleteFromDB(id) { + const col = db.get('photos'); + const exists = col.find({ id }).value(); + + if (exists) { + col.remove({ id }).write(); + console.log(` 🔴 Eliminato dal DB: ${id}`); + return true; + } + return false; + } + + return { + buildIdsListForFolder, + removeIdFromList, + deleteThumbsById, + deleteFromDB + }; +}; diff --git a/api_v1/scanner/processFile.js b/api_v1/scanner/processFile.js index 4caebc9..64cef66 100644 --- a/api_v1/scanner/processFile.js +++ b/api_v1/scanner/processFile.js @@ -8,14 +8,12 @@ const { createVideoThumbnail, createThumbnails } = require('./thumbs'); const { probeVideo } = require('./video'); const loc = require('../geo.js'); const { WEB_ROOT, PATH_FULL } = require('../config'); +const { getElevation } = require('./elevation'); + async function processFile(userName, cartella, fileRelPath, absPath, ext, st) { const isVideo = ['.mp4', '.mov', '.m4v'].includes(ext); - console.log( - `PROCESSING → user=${userName} | cartella=${cartella} | file=${fileRelPath}` - ); - const thumbBase = path.join( WEB_ROOT, 'photos', @@ -27,7 +25,10 @@ async function processFile(userName, cartella, fileRelPath, absPath, ext, st) { await fsp.mkdir(thumbBase, { recursive: true }); - const baseName = path.parse(fileRelPath).name; + // --- Nome file + estensione (dal path ASSOLUTO, sempre corretto) --- + const parsed = path.parse(absPath); + const baseName = parsed.name; // IMG_0249 + const extName = parsed.ext; // .JPG const absThumbMin = path.join(thumbBase, `${baseName}_min.jpg`); const absThumbAvg = path.join(thumbBase, `${baseName}_avg.jpg`); @@ -44,54 +45,93 @@ async function processFile(userName, cartella, fileRelPath, absPath, ext, st) { tags = await ExifReader.load(absPath, { expanded: true }); } catch {} - const timeRaw = tags?.exif?.DateTimeOriginal?.value?.[0] || null; - const takenAtIso = parseExifDateUtc(timeRaw); +let timeRaw = tags?.exif?.DateTimeOriginal?.value?.[0] || null; +let takenAtIso = parseExifDateUtc(timeRaw); - // --- GPS --- - let gps = isVideo - ? await extractGpsWithExiftool(absPath) - : extractGpsFromExif(tags); +// Fallback per i video +if (isVideo) { + const fallback = new Date(st.mtimeMs).toISOString(); + takenAtIso = fallback; + timeRaw = fallback; +} - // --- DIMENSIONI --- - let width = null, height = null, duration = null; + +// --- GPS --- +let gps = null; + +if (isVideo) { + // i video usano exiftool + gps = await extractGpsWithExiftool(absPath); +} else { + // le foto usano exifreader + gps = extractGpsFromExif(tags); +} + +// --- ALTITUDINE DA SERVIZIO ESTERNO SE MANCANTE --- +if (gps && gps.lat && gps.lng) { + if (gps.alt == null) { + gps.alt = await getElevation(gps.lat, gps.lng); + } +} + + + + // --- DIMENSIONI & ROTAZIONE --- + let width = null, height = null, duration = null, duration_ms = null, rotation = 0; if (isVideo) { const info = await probeVideo(absPath); const stream = info.streams?.find(s => s.width && s.height); + if (stream) { width = stream.width; height = stream.height; + + rotation = 0; + + if (stream?.tags?.rotate) { + rotation = Number(stream.tags.rotate); + } + + const sdl = stream?.side_data_list; + if (sdl && Array.isArray(sdl)) { + const rotEntry = sdl.find(d => d.rotation !== undefined); + if (rotEntry) rotation = Number(rotEntry.rotation); + + const matrixEntry = sdl.find(d => typeof d.displaymatrix === 'string'); + if (matrixEntry) { + const match = matrixEntry.displaymatrix.match(/rotation of ([\-0-9]+) degrees/i); + if (match) rotation = Number(match[1]); + } + } + + rotation = ((rotation % 360) + 360) % 360; } + duration = info.format?.duration || null; + duration_ms = duration ? Math.round(duration * 1000) : null; + } else { try { const meta = await sharp(absPath).metadata(); width = meta.width || null; height = meta.height || null; } catch {} + + try { + const raw = + tags?.exif?.Orientation?.value ?? + tags?.image?.Orientation?.value ?? + tags?.ifd0?.Orientation?.value ?? + null; + + const val = Array.isArray(raw) ? raw[0] : raw; + + const map = { 1: 0, 3: 180, 6: 90, 8: 270 }; + rotation = map[val] ?? 0; + } catch {} } - // --- ROTAZIONE EXIF → GRADI REALI --- - let rotation = null; - try { - const raw = - tags?.exif?.Orientation?.value ?? - tags?.image?.Orientation?.value ?? - tags?.ifd0?.Orientation?.value ?? - null; - - const val = Array.isArray(raw) ? raw[0] : raw; - - const map = { - 1: 0, - 3: 180, - 6: 90, - 8: 270 - }; - - rotation = map[val] ?? null; - } catch {} - const mime_type = inferMimeFromExt(ext); const id = sha256(`${userName}/${cartella}/${fileRelPath}`); @@ -121,6 +161,7 @@ async function processFile(userName, cartella, fileRelPath, absPath, ext, st) { id, user: userName, cartella, + name: baseName + extName, // 👈 SEMPRE CORRETTO path: fullPath, thub1: fullThub1, thub2: fullThub2, @@ -130,10 +171,10 @@ async function processFile(userName, cartella, fileRelPath, absPath, ext, st) { mime_type, width, height, - rotation, // <── ROTAZIONE IN GRADI REALI + rotation, size_bytes: st.size, mtimeMs: st.mtimeMs, - duration: isVideo ? duration : null, + duration_ms: isVideo ? duration_ms : null, location }; } diff --git a/api_v1/scanner/processFile.js.ok b/api_v1/scanner/processFile.js.ok new file mode 100644 index 0000000..0f8d1a1 --- /dev/null +++ b/api_v1/scanner/processFile.js.ok @@ -0,0 +1,171 @@ +const path = require('path'); +const fsp = require('fs/promises'); +const ExifReader = require('exifreader'); +const sharp = require('sharp'); +const { sha256, inferMimeFromExt, parseExifDateUtc } = require('./utils'); +const { extractGpsFromExif, extractGpsWithExiftool } = require('./gps'); +const { createVideoThumbnail, createThumbnails } = require('./thumbs'); +const { probeVideo } = require('./video'); +const loc = require('../geo.js'); +const { WEB_ROOT, PATH_FULL } = require('../config'); + +async function processFile(userName, cartella, fileRelPath, absPath, ext, st) { + const isVideo = ['.mp4', '.mov', '.m4v'].includes(ext); + + const thumbBase = path.join( + WEB_ROOT, + 'photos', + userName, + 'thumbs', + cartella, + path.dirname(fileRelPath) + ); + + await fsp.mkdir(thumbBase, { recursive: true }); + + const baseName = path.parse(fileRelPath).name; + + const absThumbMin = path.join(thumbBase, `${baseName}_min.jpg`); + const absThumbAvg = path.join(thumbBase, `${baseName}_avg.jpg`); + + if (isVideo) { + await createVideoThumbnail(absPath, absThumbMin, absThumbAvg); + } else { + await createThumbnails(absPath, absThumbMin, absThumbAvg); + } + + // --- EXIF --- + let tags = {}; + try { + tags = await ExifReader.load(absPath, { expanded: true }); + } catch {} + + const timeRaw = tags?.exif?.DateTimeOriginal?.value?.[0] || null; + const takenAtIso = parseExifDateUtc(timeRaw); + + // --- GPS --- + let gps = isVideo + ? await extractGpsWithExiftool(absPath) + : extractGpsFromExif(tags); + + // --- DIMENSIONI & ROTAZIONE --- + let width = null, height = null, duration = null, rotation = 0; + + if (isVideo) { + const info = await probeVideo(absPath); + const stream = info.streams?.find(s => s.width && s.height); + + if (stream) { + width = stream.width; + height = stream.height; + + // --- ROTAZIONE VIDEO (robusta, compatibile Xiaomi) --- + rotation = 0; + + // 1) rotate standard + if (stream?.tags?.rotate) { + rotation = Number(stream.tags.rotate); + } + + // 2) side_data_list rotation + const sdl = stream?.side_data_list; + if (sdl && Array.isArray(sdl)) { + const rotEntry = sdl.find(d => d.rotation !== undefined); + if (rotEntry) { + rotation = Number(rotEntry.rotation); + } + + // 3) Xiaomi: displaymatrix string + const matrixEntry = sdl.find(d => typeof d.displaymatrix === 'string'); + if (matrixEntry) { + const match = matrixEntry.displaymatrix.match(/rotation of ([\-0-9]+) degrees/i); + if (match) { + rotation = Number(match[1]); + } + } + } + + // Normalizza (Xiaomi usa -90 → 270) + rotation = ((rotation % 360) + 360) % 360; + } + + duration = info.format?.duration || null; + + } else { + // FOTO + try { + const meta = await sharp(absPath).metadata(); + width = meta.width || null; + height = meta.height || null; + } catch {} + + // --- ROTAZIONE EXIF FOTO --- + try { + const raw = + tags?.exif?.Orientation?.value ?? + tags?.image?.Orientation?.value ?? + tags?.ifd0?.Orientation?.value ?? + null; + + const val = Array.isArray(raw) ? raw[0] : raw; + + const map = { + 1: 0, + 3: 180, + 6: 90, + 8: 270 + }; + + rotation = map[val] ?? 0; + } catch {} + } + + const mime_type = inferMimeFromExt(ext); + const id = sha256(`${userName}/${cartella}/${fileRelPath}`); + + const location = gps ? await loc(gps.lng, gps.lat) : null; + + // + // --- GESTIONE PATH FULL / RELATIVI --- + // + + const relPath = fileRelPath; + const relThub1 = fileRelPath.replace(/\.[^.]+$/, '_min.jpg'); + const relThub2 = fileRelPath.replace(/\.[^.]+$/, '_avg.jpg'); + + const fullPath = PATH_FULL + ? path.posix.join('/photos', userName, cartella, fileRelPath) + : relPath; + + const fullThub1 = PATH_FULL + ? path.posix.join('/photos', userName, 'thumbs', cartella, relThub1) + : relThub1; + + const fullThub2 = PATH_FULL + ? path.posix.join('/photos', userName, 'thumbs', cartella, relThub2) + : relThub2; + + return { + id, + user: userName, + cartella, + path: fullPath, + thub1: fullThub1, + thub2: fullThub2, + gps, + data: timeRaw, + taken_at: takenAtIso, + mime_type, + width, + height, + rotation, + size_bytes: st.size, + mtimeMs: st.mtimeMs, + duration: isVideo ? duration : null, + location + }; +} + +module.exports = processFile; + + diff --git a/api_v1/scanner/processFile.js.ok2 b/api_v1/scanner/processFile.js.ok2 new file mode 100644 index 0000000..baabba4 --- /dev/null +++ b/api_v1/scanner/processFile.js.ok2 @@ -0,0 +1,171 @@ +const path = require('path'); +const fsp = require('fs/promises'); +const ExifReader = require('exifreader'); +const sharp = require('sharp'); +const { sha256, inferMimeFromExt, parseExifDateUtc } = require('./utils'); +const { extractGpsFromExif, extractGpsWithExiftool } = require('./gps'); +const { createVideoThumbnail, createThumbnails } = require('./thumbs'); +const { probeVideo } = require('./video'); +const loc = require('../geo.js'); +const { WEB_ROOT, PATH_FULL } = require('../config'); + +async function processFile(userName, cartella, fileRelPath, absPath, ext, st) { + const isVideo = ['.mp4', '.mov', '.m4v'].includes(ext); + + const thumbBase = path.join( + WEB_ROOT, + 'photos', + userName, + 'thumbs', + cartella, + path.dirname(fileRelPath) + ); + + await fsp.mkdir(thumbBase, { recursive: true }); + + const baseName = path.parse(fileRelPath).name; + + const absThumbMin = path.join(thumbBase, `${baseName}_min.jpg`); + const absThumbAvg = path.join(thumbBase, `${baseName}_avg.jpg`); + + if (isVideo) { + await createVideoThumbnail(absPath, absThumbMin, absThumbAvg); + } else { + await createThumbnails(absPath, absThumbMin, absThumbAvg); + } + + // --- EXIF --- + let tags = {}; + try { + tags = await ExifReader.load(absPath, { expanded: true }); + } catch {} + + const timeRaw = tags?.exif?.DateTimeOriginal?.value?.[0] || null; + const takenAtIso = parseExifDateUtc(timeRaw); + + // --- GPS --- + let gps = isVideo + ? await extractGpsWithExiftool(absPath) + : extractGpsFromExif(tags); + + // --- DIMENSIONI & ROTAZIONE --- + let width = null, height = null, duration = null, duration_ms = null, rotation = 0; + + if (isVideo) { + const info = await probeVideo(absPath); + const stream = info.streams?.find(s => s.width && s.height); + + if (stream) { + width = stream.width; + height = stream.height; + + // --- ROTAZIONE VIDEO (robusta, compatibile Xiaomi) --- + rotation = 0; + + // 1) rotate standard + if (stream?.tags?.rotate) { + rotation = Number(stream.tags.rotate); + } + + // 2) side_data_list rotation + const sdl = stream?.side_data_list; + if (sdl && Array.isArray(sdl)) { + const rotEntry = sdl.find(d => d.rotation !== undefined); + if (rotEntry) { + rotation = Number(rotEntry.rotation); + } + + // 3) Xiaomi: displaymatrix string + const matrixEntry = sdl.find(d => typeof d.displaymatrix === 'string'); + if (matrixEntry) { + const match = matrixEntry.displaymatrix.match(/rotation of ([\-0-9]+) degrees/i); + if (match) { + rotation = Number(match[1]); + } + } + } + + // Normalizza (Xiaomi usa -90 → 270) + rotation = ((rotation % 360) + 360) % 360; + } + + duration = info.format?.duration || null; + duration_ms = duration ? Math.round(duration * 1000) : null; + + } else { + // FOTO + try { + const meta = await sharp(absPath).metadata(); + width = meta.width || null; + height = meta.height || null; + } catch {} + + // --- ROTAZIONE EXIF FOTO --- + try { + const raw = + tags?.exif?.Orientation?.value ?? + tags?.image?.Orientation?.value ?? + tags?.ifd0?.Orientation?.value ?? + null; + + const val = Array.isArray(raw) ? raw[0] : raw; + + const map = { + 1: 0, + 3: 180, + 6: 90, + 8: 270 + }; + + rotation = map[val] ?? 0; + } catch {} + } + + const mime_type = inferMimeFromExt(ext); + const id = sha256(`${userName}/${cartella}/${fileRelPath}`); + + const location = gps ? await loc(gps.lng, gps.lat) : null; + + // + // --- GESTIONE PATH FULL / RELATIVI --- + // + + const relPath = fileRelPath; + const relThub1 = fileRelPath.replace(/\.[^.]+$/, '_min.jpg'); + const relThub2 = fileRelPath.replace(/\.[^.]+$/, '_avg.jpg'); + + const fullPath = PATH_FULL + ? path.posix.join('/photos', userName, cartella, fileRelPath) + : relPath; + + const fullThub1 = PATH_FULL + ? path.posix.join('/photos', userName, 'thumbs', cartella, relThub1) + : relThub1; + + const fullThub2 = PATH_FULL + ? path.posix.join('/photos', userName, 'thumbs', cartella, relThub2) + : relThub2; + + return { + id, + user: userName, + cartella, + path: fullPath, + thub1: fullThub1, + thub2: fullThub2, + gps, + data: timeRaw, + taken_at: takenAtIso, + mime_type, + width, + height, + rotation, + size_bytes: st.size, + mtimeMs: st.mtimeMs, + // duration: isVideo ? duration : null, + duration_ms: isVideo ? duration_ms : null, + location + }; +} + +module.exports = processFile; diff --git a/api_v1/scanner/scanCartella.js b/api_v1/scanner/scanCartella.js index cc9ce05..206f26c 100644 --- a/api_v1/scanner/scanCartella.js +++ b/api_v1/scanner/scanCartella.js @@ -1,92 +1,70 @@ -// scanner/scanCartella.js -const path = require('path'); -const fsp = require('fs/promises'); -const processFile = require('./processFile'); -const { sha256 } = require('./utils'); -const { SUPPORTED_EXTS } = require('../config'); - -async function scanCartella(userName, cartella, absCartella, previousIndexTree) { - const changes = []; - - async function walk(currentAbs, relPath = '') { - console.log(`[SCAN] Entrato in cartella: ${currentAbs}`); - - let entries = []; - try { - entries = await fsp.readdir(currentAbs, { withFileTypes: true }); - } catch (err) { - console.error(`[SCAN] Impossibile leggere: ${currentAbs} - ${err.message}`); - return; - } - - for (const e of entries) { - const absPath = path.join(currentAbs, e.name); - - if (e.isDirectory()) { - await walk(absPath, path.join(relPath, e.name)); - continue; - } - - const ext = path.extname(e.name).toLowerCase(); - if (!SUPPORTED_EXTS.has(ext)) continue; - - const fileRelPath = relPath ? `${relPath}/${e.name}` : e.name; - console.log(`[SCAN] FILE TROVATO → ${fileRelPath}`); - - const id = sha256(`${userName}/${cartella}/${fileRelPath}`); - - let st; - try { - st = await fsp.stat(absPath); - } catch { - continue; - } - - const hash = sha256(`${st.size}-${st.mtimeMs}`); - const prev = previousIndexTree?.[userName]?.[cartella]?.[id]; - - console.log(`[SCAN] CHECK → id=${id}, hash=${hash}, prevHash=${prev?.hash}`); - - // 🔥 NON facciamo più SKIP qui. - // Anche se il file è invariato, lo passiamo a scanPhoto.js - // così può rimuovere l'ID da idsIndex. - if (prev && prev.hash === hash) { - console.log(`[SCAN] FILE INVARIATO (passato a scanPhoto) → ${fileRelPath}`); - - const meta = { - id, - user: userName, - cartella, - path: `/photos/${userName}/original/${cartella}/${fileRelPath}`, - _indexHash: hash, - unchanged: true - }; - - changes.push(meta); - continue; - } - - console.log(`[SCAN] PROCESSING → ${userName}/${cartella}/${fileRelPath}`); - - const meta = await processFile( - userName, - cartella, - fileRelPath, - absPath, - ext, - st - ); - - meta.id = id; - meta.path = `/photos/${userName}/original/${cartella}/${fileRelPath}`; - meta._indexHash = hash; - - changes.push(meta); - } - } - - await walk(absCartella); - return changes; -} - -module.exports = scanCartella; +// scanner/scanCartella.js +const path = require('path'); +const fsp = require('fs/promises'); +const processFile = require('./processFile'); +const { sha256 } = require('./utils'); +const { SUPPORTED_EXTS } = require('../config'); + +async function* scanCartella(userName, cartella, absCartella, previousIndexTree) { + + async function* walk(currentAbs, relPath = '') { + let entries = []; + try { + entries = await fsp.readdir(currentAbs, { withFileTypes: true }); + } catch { + return; + } + + for (const e of entries) { + const absPath = path.join(currentAbs, e.name); + + if (e.isDirectory()) { + yield* walk(absPath, path.join(relPath, e.name)); + continue; + } + + const ext = path.extname(e.name).toLowerCase(); + if (!SUPPORTED_EXTS.has(ext)) continue; + + const fileRelPath = relPath ? `${relPath}/${e.name}` : e.name; + const id = sha256(`${userName}/${cartella}/${fileRelPath}`); + + let st; + try { st = await fsp.stat(absPath); } catch { continue; } + + const hash = sha256(`${st.size}-${st.mtimeMs}`); + const prev = previousIndexTree?.[userName]?.[cartella]?.[id]; + + if (prev && prev.hash === hash) { + yield { + id, + user: userName, + cartella, + path: `/photos/${userName}/original/${cartella}/${fileRelPath}`, + _indexHash: hash, + unchanged: true + }; + continue; + } + + const meta = await processFile( + userName, + cartella, + fileRelPath, + absPath, + ext, + st + ); + + meta.id = id; + meta.path = `/photos/${userName}/original/${cartella}/${fileRelPath}`; + meta._indexHash = hash; + + yield meta; + } + } + + yield* walk(absCartella); +} + +module.exports = scanCartella; diff --git a/api_v1/scanner/scanCartella.js.ok b/api_v1/scanner/scanCartella.js.ok index c57b048..c949e13 100644 --- a/api_v1/scanner/scanCartella.js.ok +++ b/api_v1/scanner/scanCartella.js.ok @@ -1,78 +1,72 @@ -// scanner/scanCartella.js -const path = require('path'); -const fsp = require('fs/promises'); -const processFile = require('./processFile'); -const { sha256 } = require('./utils'); -const { SUPPORTED_EXTS } = require('../config'); - -async function scanCartella(userName, cartella, absCartella, previousIndexTree) { - const changes = []; - - async function walk(currentAbs, relPath = '') { - console.log(`[SCAN] Entrato in cartella: ${currentAbs}`); - - let entries = []; - try { - entries = await fsp.readdir(currentAbs, { withFileTypes: true }); - } catch (err) { - console.error(`[SCAN] Impossibile leggere: ${currentAbs} - ${err.message}`); - return; - } - - for (const e of entries) { - const absPath = path.join(currentAbs, e.name); - - if (e.isDirectory()) { - await walk(absPath, path.join(relPath, e.name)); - continue; - } - - const ext = path.extname(e.name).toLowerCase(); - if (!SUPPORTED_EXTS.has(ext)) continue; - - const fileRelPath = relPath ? `${relPath}/${e.name}` : e.name; - console.log(`[SCAN] FILE TROVATO → ${fileRelPath}`); - - const id = sha256(`${userName}/${cartella}/${fileRelPath}`); - - let st; - try { - st = await fsp.stat(absPath); - } catch { - continue; - } - - const hash = sha256(`${st.size}-${st.mtimeMs}`); - const prev = previousIndexTree?.[userName]?.[cartella]?.[id]; - - console.log(`[SCAN] CHECK → id=${id}, hash=${hash}, prevHash=${prev?.hash}`); - - if (prev && prev.hash === hash) { - console.log(`[SCAN] SKIP (unchanged) → ${fileRelPath}`); - continue; - } - - console.log(`[SCAN] PROCESSING → ${userName}/${cartella}/${fileRelPath}`); - - const meta = await processFile( - userName, - cartella, - fileRelPath, - absPath, - ext, - st - ); - - meta.id = id; - meta.path = `/photos/${userName}/original/${cartella}/${fileRelPath}`; - meta._indexHash = hash; - - changes.push(meta); - } - } - - await walk(absCartella); - return changes; -} - -module.exports = scanCartella; +// scanner/scanCartella.js +const path = require('path'); +const fsp = require('fs/promises'); +const processFile = require('./processFile'); +const { sha256 } = require('./utils'); +const { SUPPORTED_EXTS } = require('../config'); + +async function* scanCartella(userName, cartella, absCartella, previousIndexTree) { + + async function* walk(currentAbs, relPath = '') { + let entries = []; + try { + entries = await fsp.readdir(currentAbs, { withFileTypes: true }); + } catch { + return; + } + + for (const e of entries) { + const absPath = path.join(currentAbs, e.name); + + if (e.isDirectory()) { + yield* walk(absPath, path.join(relPath, e.name)); + continue; + } + + const ext = path.extname(e.name).toLowerCase(); + if (!SUPPORTED_EXTS.has(ext)) continue; + + const fileRelPath = relPath ? `${relPath}/${e.name}` : e.name; + const id = sha256(`${userName}/${cartella}/${fileRelPath}`); + + let st; + try { st = await fsp.stat(absPath); } catch { continue; } + + const hash = sha256(`${st.size}-${st.mtimeMs}`); + const prev = previousIndexTree?.[userName]?.[cartella]?.[id]; + + // FILE INVARIATO + if (prev && prev.hash === hash) { + yield { + id, + user: userName, + cartella, + path: `/photos/${userName}/original/${cartella}/${fileRelPath}`, + _indexHash: hash, + unchanged: true + }; + continue; + } + + // FILE NUOVO/MODIFICATO + const meta = await processFile( + userName, + cartella, + fileRelPath, + absPath, + ext, + st + ); + + meta.id = id; + meta.path = `/photos/${userName}/original/${cartella}/${fileRelPath}`; + meta._indexHash = hash; + + yield meta; + } + } + + yield* walk(absCartella); +} + +module.exports = scanCartella; diff --git a/api_v1/scanner/scanCartella.js.new b/api_v1/scanner/scanCartella.js.ok.con_log_ivariati similarity index 57% rename from api_v1/scanner/scanCartella.js.new rename to api_v1/scanner/scanCartella.js.ok.con_log_ivariati index c57b048..83ef76a 100644 --- a/api_v1/scanner/scanCartella.js.new +++ b/api_v1/scanner/scanCartella.js.ok.con_log_ivariati @@ -4,18 +4,15 @@ const fsp = require('fs/promises'); const processFile = require('./processFile'); const { sha256 } = require('./utils'); const { SUPPORTED_EXTS } = require('../config'); +const { log } = require('./logger'); -async function scanCartella(userName, cartella, absCartella, previousIndexTree) { - const changes = []; - - async function walk(currentAbs, relPath = '') { - console.log(`[SCAN] Entrato in cartella: ${currentAbs}`); +async function* scanCartella(userName, cartella, absCartella, previousIndexTree) { + async function* walk(currentAbs, relPath = '') { let entries = []; try { entries = await fsp.readdir(currentAbs, { withFileTypes: true }); - } catch (err) { - console.error(`[SCAN] Impossibile leggere: ${currentAbs} - ${err.message}`); + } catch { return; } @@ -23,7 +20,7 @@ async function scanCartella(userName, cartella, absCartella, previousIndexTree) const absPath = path.join(currentAbs, e.name); if (e.isDirectory()) { - await walk(absPath, path.join(relPath, e.name)); + yield* walk(absPath, path.join(relPath, e.name)); continue; } @@ -31,29 +28,26 @@ async function scanCartella(userName, cartella, absCartella, previousIndexTree) if (!SUPPORTED_EXTS.has(ext)) continue; const fileRelPath = relPath ? `${relPath}/${e.name}` : e.name; - console.log(`[SCAN] FILE TROVATO → ${fileRelPath}`); - const id = sha256(`${userName}/${cartella}/${fileRelPath}`); let st; - try { - st = await fsp.stat(absPath); - } catch { - continue; - } + try { st = await fsp.stat(absPath); } catch { continue; } const hash = sha256(`${st.size}-${st.mtimeMs}`); const prev = previousIndexTree?.[userName]?.[cartella]?.[id]; - console.log(`[SCAN] CHECK → id=${id}, hash=${hash}, prevHash=${prev?.hash}`); - if (prev && prev.hash === hash) { - console.log(`[SCAN] SKIP (unchanged) → ${fileRelPath}`); + yield { + id, + user: userName, + cartella, + path: `/photos/${userName}/original/${cartella}/${fileRelPath}`, + _indexHash: hash, + unchanged: true + }; continue; } - console.log(`[SCAN] PROCESSING → ${userName}/${cartella}/${fileRelPath}`); - const meta = await processFile( userName, cartella, @@ -67,12 +61,11 @@ async function scanCartella(userName, cartella, absCartella, previousIndexTree) meta.path = `/photos/${userName}/original/${cartella}/${fileRelPath}`; meta._indexHash = hash; - changes.push(meta); + yield meta; } } - await walk(absCartella); - return changes; + yield* walk(absCartella); } module.exports = scanCartella; diff --git a/api_v1/scanner/scanCartella.js.ok.senza_log_invariati b/api_v1/scanner/scanCartella.js.ok.senza_log_invariati new file mode 100644 index 0000000..206f26c --- /dev/null +++ b/api_v1/scanner/scanCartella.js.ok.senza_log_invariati @@ -0,0 +1,70 @@ +// scanner/scanCartella.js +const path = require('path'); +const fsp = require('fs/promises'); +const processFile = require('./processFile'); +const { sha256 } = require('./utils'); +const { SUPPORTED_EXTS } = require('../config'); + +async function* scanCartella(userName, cartella, absCartella, previousIndexTree) { + + async function* walk(currentAbs, relPath = '') { + let entries = []; + try { + entries = await fsp.readdir(currentAbs, { withFileTypes: true }); + } catch { + return; + } + + for (const e of entries) { + const absPath = path.join(currentAbs, e.name); + + if (e.isDirectory()) { + yield* walk(absPath, path.join(relPath, e.name)); + continue; + } + + const ext = path.extname(e.name).toLowerCase(); + if (!SUPPORTED_EXTS.has(ext)) continue; + + const fileRelPath = relPath ? `${relPath}/${e.name}` : e.name; + const id = sha256(`${userName}/${cartella}/${fileRelPath}`); + + let st; + try { st = await fsp.stat(absPath); } catch { continue; } + + const hash = sha256(`${st.size}-${st.mtimeMs}`); + const prev = previousIndexTree?.[userName]?.[cartella]?.[id]; + + if (prev && prev.hash === hash) { + yield { + id, + user: userName, + cartella, + path: `/photos/${userName}/original/${cartella}/${fileRelPath}`, + _indexHash: hash, + unchanged: true + }; + continue; + } + + const meta = await processFile( + userName, + cartella, + fileRelPath, + absPath, + ext, + st + ); + + meta.id = id; + meta.path = `/photos/${userName}/original/${cartella}/${fileRelPath}`; + meta._indexHash = hash; + + yield meta; + } + } + + yield* walk(absCartella); +} + +module.exports = scanCartella; diff --git a/api_v1/scanner/scanPhoto.js b/api_v1/scanner/scanPhoto.js index 563b909..084dcbd 100644 --- a/api_v1/scanner/scanPhoto.js +++ b/api_v1/scanner/scanPhoto.js @@ -1,8 +1,8 @@ // scanner/scanPhoto.js const path = require('path'); -const fs = require('fs'); const fsp = require('fs/promises'); +const { log } = require('./logger'); const { loadPreviousIndex, saveIndex } = require('./indexStore'); const scanCartella = require('./scanCartella'); const postWithAuth = require('./postWithAuth'); @@ -11,13 +11,59 @@ const { WEB_ROOT, SEND_PHOTOS, BASE_URL, - WRITE_INDEX + WRITE_INDEX, + SUPPORTED_EXTS } = require('../config'); const createCleanupFunctions = require('./orphanCleanup'); +// Variabile per log completo o ridotto +const LOG_VERBOSE = (process.env.LOG_VERBOSE === "true"); + +// Formatta ms → HH:MM:SS +function formatTime(ms) { + const sec = Math.floor(ms / 1000); + const h = String(Math.floor(sec / 3600)).padStart(2, '0'); + const m = String(Math.floor((sec % 3600) / 60)).padStart(2, '0'); + const s = String(sec % 60).padStart(2, '0'); + return `${h}:${m}:${s}`; +} + +// --------------------------------------------------------- +// ⚡ CONTEGGIO FILE VELOCE (senza hashing, EXIF, sharp, ecc.) +// --------------------------------------------------------- +async function countFilesFast(rootDir) { + let count = 0; + + async function walk(dir) { + let entries = []; + try { + entries = await fsp.readdir(dir, { withFileTypes: true }); + } catch { + return; + } + + for (const e of entries) { + const abs = path.join(dir, e.name); + + if (e.isDirectory()) { + await walk(abs); + } else { + const ext = path.extname(e.name).toLowerCase(); + if (SUPPORTED_EXTS.has(ext)) { + count++; + } + } + } + } + + await walk(rootDir); + return count; +} + async function scanPhoto(dir, userName, db) { - console.log(`[SCAN] Avvio scanPhoto per user=${userName}`); + const start = Date.now(); + log(`🔵 Inizio scan globale per user=${userName}`); const previousIndexTree = await loadPreviousIndex(); const nextIndexTree = JSON.parse(JSON.stringify(previousIndexTree || {})); @@ -32,73 +78,80 @@ async function scanPhoto(dir, userName, db) { const photosRoot = path.resolve(__dirname, '..', '..', WEB_ROOT, 'photos'); const userDir = path.join(photosRoot, userName, 'original'); - let changes = []; + let totalNew = 0; + let totalDeleted = 0; + let totalUnchanged = 0; + + let newFiles = []; // --------------------------------------------------------- - // SCAN DI UNA SINGOLA CARTELLA + // 1) ⚡ CONTEGGIO FILE SUPER VELOCE + // --------------------------------------------------------- + const TOTAL_FILES = await countFilesFast(userDir); + + if (TOTAL_FILES === 0) { + log(`Nessun file trovato per user=${userName}`); + return []; + } + + log(`📦 File totali da processare: ${TOTAL_FILES}`); + + let CURRENT = 0; + + // --------------------------------------------------------- + // Funzione per aggiornare il file statico leggibile dall’HTML + // --------------------------------------------------------- + async function updateStatusFile() { + const now = Date.now(); + const elapsedMs = now - start; + const avg = elapsedMs / CURRENT; + const remainingMs = (TOTAL_FILES - CURRENT) * avg; + + const status = { + current: CURRENT, + total: TOTAL_FILES, + percent: Number((CURRENT / TOTAL_FILES * 100).toFixed(2)), + eta: formatTime(remainingMs), + elapsed: formatTime(elapsedMs) + }; + + const statusPath = path.resolve(__dirname, "..", "..", "public/photos/scan_status.json"); + await fsp.writeFile(statusPath, JSON.stringify(status)); + } + + // --------------------------------------------------------- + // 2) SCAN REALE DELLE CARTELLE // --------------------------------------------------------- async function scanSingleFolder(user, cartella, absCartella) { - console.log(`\n==============================================`); - console.log(`[SCAN] CARTELLA → ${user}/${cartella}`); - console.log(`[DEBUG] SCANSIONE CARTELLA PATH: ${absCartella}`); - console.log("=============================================="); + log(`📁 Inizio cartella: ${cartella}`); let idsIndex = await buildIdsListForFolder(user, cartella); - const folderFiles = await scanCartella( - user, - cartella, - absCartella, - previousIndexTree - ); + let newCount = 0; + let unchangedCount = 0; + let deletedCount = 0; - for (const m of folderFiles) { - const prev = previousIndexTree?.[m.user]?.[m.cartella]?.[m.id]; + for await (const m of scanCartella(user, cartella, absCartella, previousIndexTree)) { + CURRENT++; + const fileName = m.path.split('/').pop(); + const prefix = `[${CURRENT}/${TOTAL_FILES}]`; - // 🔥 FILE INVARIATO - if (prev && prev.hash === m._indexHash) { + // Aggiorna file statico per l’HTML + await updateStatusFile(); - console.log("\n================ FILE INVARIATO ================"); - console.log("[FILE]:", m.path); - - console.log("[ID OGGI]:", JSON.stringify(m.id)); - console.log("[ID INDEX]:", JSON.stringify(prev.id)); - console.log("[UGUALI?]:", m.id === prev.id); - - console.log("[LEN OGGI]:", m.id.length); - console.log("[LEN INDEX]:", prev.id.length); - - console.log("----- BYTE PER BYTE -----"); - const max = Math.min(m.id.length, prev.id.length, 64); - for (let i = 0; i < max; i++) { - const a = m.id[i]; - const b = prev.id[i]; - console.log( - i.toString().padStart(2, "0"), - a, - b, - a === b ? "==" : "!=" - ); + // ⚪ FILE INVARIATO + if (m.unchanged) { + if (LOG_VERBOSE) { + log(`${prefix} ⚪ Invariato: ${fileName}`); } - - console.log("----- idsIndex PRIMA -----"); - console.log(idsIndex.map(x => JSON.stringify(x))); - - const found = idsIndex.includes(m.id); - console.log("[ID PRESENTE IN idsIndex?]:", found); - - const newList = idsIndex.filter(x => x !== m.id); - - console.log("----- idsIndex DOPO -----"); - console.log(newList.map(x => JSON.stringify(x))); - - idsIndex = newList; - - console.log("=================================================\n"); + idsIndex = removeIdFromList(idsIndex, m.id); + unchangedCount++; continue; } - // 🔥 FILE NUOVO O MODIFICATO + // 🟢 FILE NUOVO O MODIFICATO + log(`${prefix} 🟢 Nuovo/Modificato: ${fileName}`); + nextIndexTree[m.user] ??= {}; nextIndexTree[m.user][m.cartella] ??= {}; nextIndexTree[m.user][m.cartella][m.id] = { @@ -110,38 +163,33 @@ async function scanPhoto(dir, userName, db) { }; idsIndex = removeIdFromList(idsIndex, m.id); - changes.push(m); - } - - // --------------------------------------------------------- - // ORFANI - // --------------------------------------------------------- - if (idsIndex.length > 0) { - console.log(`[ORPHAN] ID orfani trovati nella cartella ${cartella}:`); - idsIndex.forEach(id => console.log(" -", id)); - } else { - console.log(`[ORPHAN] Nessun ID orfano nella cartella ${cartella}`); + newCount++; + newFiles.push(m); } + // 🔴 ORFANI for (const orphanId of idsIndex) { - console.log(`\n[ORPHAN] Eliminazione ID → ${orphanId}`); + const old = previousIndexTree?.[user]?.[cartella]?.[orphanId]; + const fileName = old?.path?.split('/').pop() || orphanId; - const thumbsDeleted = await deleteThumbsById(orphanId); - console.log(` → thumbsDeleted = ${thumbsDeleted}`); + log(` 🔴 Eliminato: ${fileName}`); - const dbDeleted = deleteFromDB(orphanId); - console.log(` → dbDeleted = ${dbDeleted}`); + await deleteThumbsById(orphanId); + deleteFromDB(orphanId); const userTree = nextIndexTree[user]; - if (userTree && userTree[cartella] && userTree[cartella][orphanId]) { + if (userTree?.[cartella]?.[orphanId]) { delete userTree[cartella][orphanId]; - console.log(` → nextIndexTree updated (removed ${orphanId})`); } - console.log(`[ORPHAN] COMPLETATO → ${orphanId}`); + deletedCount++; } - console.log(`==============================================\n`); + log(`📊 Fine cartella ${cartella}: invariati=${unchangedCount}, nuovi=${newCount}, cancellati=${deletedCount}`); + + totalNew += newCount; + totalDeleted += deletedCount; + totalUnchanged += unchangedCount; } // --------------------------------------------------------- @@ -151,7 +199,7 @@ async function scanPhoto(dir, userName, db) { try { entries = await fsp.readdir(userDir, { withFileTypes: true }); } catch { - console.log(`[SCAN] Nessuna directory per utente ${userName}`); + log(`Nessuna directory per utente ${userName}`); return []; } @@ -170,20 +218,30 @@ async function scanPhoto(dir, userName, db) { } // --------------------------------------------------------- - // INVIO AL SERVER + // INVIO AL SERVER / POPOLAZIONE DB // --------------------------------------------------------- - if (SEND_PHOTOS && BASE_URL && changes.length) { - for (const p of changes) { + if (SEND_PHOTOS && BASE_URL && newFiles.length > 0) { + for (const p of newFiles) { + const fileName = p.path.split('/').pop(); + try { await postWithAuth(`${BASE_URL}/photos`, p); + log(`📤 Inviato al server: ${fileName}`); } catch (err) { - console.error('Errore invio:', err.message); + log(`Errore invio ${fileName}: ${err.message}`); } } } - console.log(`SCAN COMPLETATA: ${changes.length} file aggiornati`); - return changes; + // --------------------------------------------------------- + // FINE SCAN + // --------------------------------------------------------- + const elapsed = ((Date.now() - start) / 1000).toFixed(2); + + log(`🟣 Scan COMPLETATO: nuovi=${totalNew}, cancellati=${totalDeleted}, invariati=${totalUnchanged}`); + log(`⏱ Tempo totale: ${elapsed}s`); + + return newFiles; } module.exports = scanPhoto; diff --git a/api_v1/scanner/scanPhoto.js.new b/api_v1/scanner/scanPhoto.js.new deleted file mode 100644 index b7b6d54..0000000 --- a/api_v1/scanner/scanPhoto.js.new +++ /dev/null @@ -1,176 +0,0 @@ -// scanner/scanPhoto.js -const path = require('path'); -const fs = require('fs'); -const fsp = require('fs/promises'); - -const { loadPreviousIndex, saveIndex } = require('./indexStore'); -const scanCartella = require('./scanCartella'); -const postWithAuth = require('./postWithAuth'); - -const { - WEB_ROOT, - SEND_PHOTOS, - BASE_URL, - WRITE_INDEX -} = require('../config'); - -const createCleanupFunctions = require('./orphanCleanup'); - -async function scanPhoto(dir, userName, db) { - console.log(`[SCAN] Avvio scanPhoto per user=${userName}`); - - // Log di configurazione (utile per diagnosi) - console.log('[SCAN] Config:', { - WEB_ROOT, - SEND_PHOTOS, - BASE_URL, - WRITE_INDEX - }); - - // 1) Carico l’indice precedente (fonte unica durante la run) - const previousIndexTree = await loadPreviousIndex(); // <— RAM source - // 2) Clono l’indice per costruire il nuovo - const nextIndexTree = JSON.parse(JSON.stringify(previousIndexTree || {})); - - // Funzioni di cleanup (solo thumbs + DB) - // NOTA: con Opzione A NON leggiamo più l'indice da file e NON aggiorniamo index.json "in itinere" - const { - // buildIdsListForFolder, // <— NON usato in Opzione A - removeIdFromList, - deleteThumbsById, - deleteFromDB - // deleteFromIndexById // <— NON usato in Opzione A - } = createCleanupFunctions(db); // [1](https://electrolux-my.sharepoint.com/personal/fabio_micheluz_electrolux_com/Documents/File%20di%20Microsoft%20Copilot%20Chat/orphanCleanup.js.txt) - - const photosRoot = path.resolve(__dirname, '..', '..', WEB_ROOT, 'photos'); - const userDir = path.join(photosRoot, userName, 'original'); - - let changes = []; - - // --------------------------------------------------------- - // SCAN DI UNA SINGOLA CARTELLA - // --------------------------------------------------------- - async function scanSingleFolder(user, cartella, absCartella) { - console.log(`\n==============================================`); - console.log(`[SCAN] CARTELLA → ${user}/${cartella}`); - console.log("=============================================="); - - // Costruisci idsIndex dall'indice in RAM (previousIndexTree) - // Fonte unica di verità per questa run → evita drift con il file su disco. - let idsIndex = Object.keys(previousIndexTree?.[user]?.[cartella] || {}) - .filter(k => k !== '_folderHash'); // <— niente metadati - console.log(`[ORPHAN] ID iniziali in RAM (${idsIndex.length}):`); - idsIndex.forEach(id => console.log(" -", id)); - - // File reali trovati nella cartella - const folderFiles = await scanCartella( - user, - cartella, - absCartella, - previousIndexTree - ); // [2](https://electrolux-my.sharepoint.com/personal/fabio_micheluz_electrolux_com/Documents/File%20di%20Microsoft%20Copilot%20Chat/scanCartella.js.txt) - - for (const m of folderFiles) { - const prev = previousIndexTree?.[m.user]?.[m.cartella]?.[m.id]; - - // 🔥 FILE INVARIATO - if (prev && prev.hash === m._indexHash) { - console.log(`[SCAN] SKIP (unchanged) → ${m.path}`); - // Rimuovo SOLO dalla lista orfani (dalla RAM) - idsIndex = removeIdFromList(idsIndex, m.id); // [1](https://electrolux-my.sharepoint.com/personal/fabio_micheluz_electrolux_com/Documents/File%20di%20Microsoft%20Copilot%20Chat/orphanCleanup.js.txt) - continue; - } - - // 🔥 FILE NUOVO O MODIFICATO → aggiorno nextIndexTree - nextIndexTree[m.user] ??= {}; - nextIndexTree[m.user][m.cartella] ??= {}; - nextIndexTree[m.user][m.cartella][m.id] = { - id: m.id, - user: m.user, - cartella: m.cartella, - path: m.path, - hash: m._indexHash - }; - - // Rimuovo dalla lista orfani (in RAM) - idsIndex = removeIdFromList(idsIndex, m.id); // [1](https://electrolux-my.sharepoint.com/personal/fabio_micheluz_electrolux_com/Documents/File%20di%20Microsoft%20Copilot%20Chat/orphanCleanup.js.txt) - changes.push(m); - } - - // --------------------------------------------------------- - // ORFANI → rimuovo SOLO da nextIndexTree + thumbs + DB - // --------------------------------------------------------- - if (idsIndex.length > 0) { - console.log(`[ORPHAN] ID orfani trovati nella cartella ${cartella}:`); - idsIndex.forEach(id => console.log(" -", id)); - } else { - console.log(`[ORPHAN] Nessun ID orfano nella cartella ${cartella}`); - } - - for (const orphanId of idsIndex) { - console.log(`\n[ORPHAN] Eliminazione ID → ${orphanId}`); - - // 1) Cancello thumbs - const thumbsDeleted = await deleteThumbsById(orphanId); // [1](https://electrolux-my.sharepoint.com/personal/fabio_micheluz_electrolux_com/Documents/File%20di%20Microsoft%20Copilot%20Chat/orphanCleanup.js.txt) - console.log(` → thumbsDeleted = ${thumbsDeleted}`); - - // 2) Cancello dal DB - const dbDeleted = deleteFromDB(orphanId); // [1](https://electrolux-my.sharepoint.com/personal/fabio_micheluz_electrolux_com/Documents/File%20di%20Microsoft%20Copilot%20Chat/orphanCleanup.js.txt) - console.log(` → dbDeleted = ${dbDeleted}`); - - // 3) Cancello SOLO da nextIndexTree (in RAM) - const userTree = nextIndexTree[user]; - if (userTree && userTree[cartella] && userTree[cartella][orphanId]) { - delete userTree[cartella][orphanId]; - console.log(` → nextIndexTree updated (removed ${orphanId})`); - } - - console.log(`[ORPHAN] COMPLETATO → ${orphanId}`); - } - - console.log(`==============================================\n`); - } - - // --------------------------------------------------------- - // SCAN SOLO SOTTOCARTELLE — niente _root - // --------------------------------------------------------- - let entries = []; - try { - entries = await fsp.readdir(userDir, { withFileTypes: true }); - } catch { - console.log(`[SCAN] Nessuna directory per utente ${userName}`); - return []; - } - - for (const e of entries) { - if (!e.isDirectory()) continue; - const cartella = e.name; - const absCartella = path.join(userDir, cartella); - await scanSingleFolder(userName, cartella, absCartella); - } - - // --------------------------------------------------------- - // SALVO SOLO nextIndexTree (scrittura unica a fine run) - // --------------------------------------------------------- - if (WRITE_INDEX) { - await saveIndex(nextIndexTree); // [1](https://electrolux-my.sharepoint.com/personal/fabio_micheluz_electrolux_com/Documents/File%20di%20Microsoft%20Copilot%20Chat/orphanCleanup.js.txt) - } - - // --------------------------------------------------------- - // INVIO AL SERVER - // --------------------------------------------------------- - if (SEND_PHOTOS && BASE_URL && changes.length) { - for (const p of changes) { - try { - await postWithAuth(`${BASE_URL}/photos`, p); - } catch (err) { - console.error('Errore invio:', err.message); - } - } - } - - console.log(`SCAN COMPLETATA: ${changes.length} file aggiornati`); - return changes; -} - -module.exports = scanPhoto; diff --git a/api_v1/scanner/scanPhoto.js.ok b/api_v1/scanner/scanPhoto.js.ok index a6ef7cc..7c36b27 100644 --- a/api_v1/scanner/scanPhoto.js.ok +++ b/api_v1/scanner/scanPhoto.js.ok @@ -1,153 +1,165 @@ -// scanner/scanPhoto.js -const path = require('path'); -const fsp = require('fs/promises'); -const { loadPreviousIndex, saveIndex } = require('./indexStore'); -const scanUserRoot = require('./scanUser'); -const postWithAuth = require('./postWithAuth'); -const { WEB_ROOT, SEND_PHOTOS, BASE_URL, WRITE_INDEX } = require('../config'); - -const COMMON = 'Common'; - -async function scanPhoto(dir, userName) { - console.log(`[SCAN] Avvio scanPhoto per user=${userName}`); - - const previousIndexTree = await loadPreviousIndex(); - console.log(`[SCAN] previousIndexTree keys:`, Object.keys(previousIndexTree)); - - const nextIndexTree = JSON.parse(JSON.stringify(previousIndexTree || {})); - const photosRoot = path.resolve(__dirname, '..', '..', WEB_ROOT, 'photos'); - - console.log(`[SCAN] photosRoot = ${photosRoot}`); - - let changes = []; - - // --------------------------------------------------------- - // 1) ADMIN → scansiona tutti gli utenti - // --------------------------------------------------------- - if (userName === 'Admin') { - let entries = []; - try { - entries = await fsp.readdir(photosRoot, { withFileTypes: true }); - } catch { - console.error(`[SCAN] photosRoot non accessibile`); - return []; - } - - console.log(`[SCAN] Trovati ${entries.length} utenti/cartelle`); - - for (const e of entries) { - if (!e.isDirectory()) continue; - - const user = e.name; - if (user.toLowerCase() === COMMON.toLowerCase()) continue; - - const userDir = path.join(photosRoot, user, 'original'); - - try { - const st = await fsp.stat(userDir); - if (!st.isDirectory()) continue; - } catch { - continue; - } - - console.log(`[SCAN] → Scansiono utente: ${user}`); - - const userChanges = await scanUserRoot(user, userDir, previousIndexTree); - - for (const m of userChanges) { - nextIndexTree[m.user] ??= {}; - nextIndexTree[m.user][m.cartella] ??= {}; - nextIndexTree[m.user][m.cartella][m.id] = { - id: m.id, - user: m.user, - cartella: m.cartella, - path: m.path, - hash: m._indexHash - }; - } - - changes.push(...userChanges); - } - - // Common - const commonDir = path.join(photosRoot, COMMON, 'original'); - try { - const st = await fsp.stat(commonDir); - if (st.isDirectory()) { - console.log(`[SCAN] → Scansiono Common`); - const commonChanges = await scanUserRoot(COMMON, commonDir, previousIndexTree); - - for (const m of commonChanges) { - nextIndexTree[COMMON] ??= {}; - nextIndexTree[COMMON][m.cartella] ??= {}; - nextIndexTree[COMMON][m.cartella][m.id] = { - id: m.id, - user: m.user, - cartella: m.cartella, - path: m.path, - hash: m._indexHash - }; - } - - changes.push(...commonChanges); - } - } catch {} - } - - // --------------------------------------------------------- - // 2) NON-ADMIN → scansiona SOLO l’utente richiesto - // --------------------------------------------------------- - else { - const userDir = path.join(photosRoot, userName, 'original'); - - try { - const st = await fsp.stat(userDir); - if (st.isDirectory()) { - console.log(`[SCAN] → Scansiono utente singolo: ${userName}`); - - const userChanges = await scanUserRoot(userName, userDir, previousIndexTree); - - for (const m of userChanges) { - nextIndexTree[m.user] ??= {}; - nextIndexTree[m.user][m.cartella] ??= {}; - nextIndexTree[m.user][m.cartella][m.id] = { - id: m.id, - user: m.user, - cartella: m.cartella, - path: m.path, - hash: m._indexHash - }; - } - - changes.push(...userChanges); - } - } catch { - console.log(`[SCAN] Nessuna directory per utente ${userName}`); - } - } - - // --------------------------------------------------------- - // SALVA INDEX - // --------------------------------------------------------- - if (WRITE_INDEX) { - await saveIndex(nextIndexTree); - } - - // --------------------------------------------------------- - // INVIA AL SERVER - // --------------------------------------------------------- - if (SEND_PHOTOS && BASE_URL && changes.length) { - for (const p of changes) { - try { - await postWithAuth(`${BASE_URL}/photos`, p); - } catch (err) { - console.error('Errore invio:', err.message); - } - } - } - - console.log(`SCAN COMPLETATA: ${changes.length} file aggiornati`); - return changes; -} - -module.exports = scanPhoto; +// scanner/scanPhoto.js +const path = require('path'); +const fsp = require('fs/promises'); + +const { loadPreviousIndex, saveIndex } = require('./indexStore'); +const scanCartella = require('./scanCartella'); +const postWithAuth = require('./postWithAuth'); + +const { + WEB_ROOT, + SEND_PHOTOS, + BASE_URL, + WRITE_INDEX +} = require('../config'); + +const createCleanupFunctions = require('./orphanCleanup'); + +async function scanPhoto(dir, userName, db) { + const start = Date.now(); + console.log(`\n🔵 Inizio scan globale per user=${userName}`); + + const previousIndexTree = await loadPreviousIndex(); + const nextIndexTree = JSON.parse(JSON.stringify(previousIndexTree || {})); + + const { + buildIdsListForFolder, + removeIdFromList, + deleteThumbsById, + deleteFromDB + } = createCleanupFunctions(db); + + const photosRoot = path.resolve(__dirname, '..', '..', WEB_ROOT, 'photos'); + const userDir = path.join(photosRoot, userName, 'original'); + + let totalNew = 0; + let totalDeleted = 0; + let totalUnchanged = 0; + + // 🔥 Array dei file nuovi/modificati (per DB e server) + let newFiles = []; + + // --------------------------------------------------------- + // SCAN DI UNA SINGOLA CARTELLA + // --------------------------------------------------------- + async function scanSingleFolder(user, cartella, absCartella) { + console.log(`\n📁 Inizio cartella: ${cartella}`); + + let idsIndex = await buildIdsListForFolder(user, cartella); + + let newCount = 0; + let unchangedCount = 0; + let deletedCount = 0; + + // STREAMING DEI FILE + for await (const m of scanCartella(user, cartella, absCartella, previousIndexTree)) { + const fileName = m.path.split('/').pop(); + + // ⚪ FILE INVARIATO + if (m.unchanged) { + console.log(` ⚪ Invariato: ${fileName}`); + idsIndex = removeIdFromList(idsIndex, m.id); + unchangedCount++; + continue; + } + + // 🟢 FILE NUOVO O MODIFICATO + console.log(` 🟢 Nuovo/Modificato: ${fileName}`); + + nextIndexTree[m.user] ??= {}; + nextIndexTree[m.user][m.cartella] ??= {}; + nextIndexTree[m.user][m.cartella][m.id] = { + id: m.id, + user: m.user, + cartella: m.cartella, + path: m.path, + hash: m._indexHash + }; + + idsIndex = removeIdFromList(idsIndex, m.id); + newCount++; + + // Aggiungiamo alla lista dei file nuovi + newFiles.push(m); + } + + // 🔴 ORFANI (FILE CANCELLATI) + for (const orphanId of idsIndex) { + const old = previousIndexTree?.[user]?.[cartella]?.[orphanId]; + const fileName = old?.path?.split('/').pop() || orphanId; + + console.log(` 🔴 Eliminato: ${fileName}`); + + await deleteThumbsById(orphanId); + deleteFromDB(orphanId); + + const userTree = nextIndexTree[user]; + if (userTree?.[cartella]?.[orphanId]) { + delete userTree[cartella][orphanId]; + } + + deletedCount++; + } + + console.log( + `📊 Fine cartella ${cartella}: invariati=${unchangedCount}, nuovi=${newCount}, cancellati=${deletedCount}` + ); + + totalNew += newCount; + totalDeleted += deletedCount; + totalUnchanged += unchangedCount; + } + + // --------------------------------------------------------- + // SCAN SOTTOCARTELLE + // --------------------------------------------------------- + let entries = []; + try { + entries = await fsp.readdir(userDir, { withFileTypes: true }); + } catch { + console.log(`Nessuna directory per utente ${userName}`); + return []; + } + + for (const e of entries) { + if (!e.isDirectory()) continue; + const cartella = e.name; + const absCartella = path.join(userDir, cartella); + await scanSingleFolder(userName, cartella, absCartella); + } + + // --------------------------------------------------------- + // SALVO INDEX + // --------------------------------------------------------- + if (WRITE_INDEX) { + await saveIndex(nextIndexTree); + } + + // --------------------------------------------------------- + // INVIO AL SERVER / POPOLAZIONE DB + // --------------------------------------------------------- + if (SEND_PHOTOS && BASE_URL && newFiles.length > 0) { + for (const p of newFiles) { + const fileName = p.path.split('/').pop(); + + try { + await postWithAuth(`${BASE_URL}/photos`, p); + console.log(`📤 Inviato al server: ${fileName}`); + } catch (err) { + console.error(`Errore invio ${fileName}:`, err.message); + } + } + } + + // --------------------------------------------------------- + // FINE SCAN + // --------------------------------------------------------- + const elapsed = ((Date.now() - start) / 1000).toFixed(2); + + console.log(`\n🟣 Scan COMPLETATO: nuovi=${totalNew}, cancellati=${totalDeleted}, invariati=${totalUnchanged}`); + console.log(`⏱ Tempo totale: ${elapsed}s\n`); + + return newFiles; +} + +module.exports = scanPhoto; diff --git a/api_v1/scanner/scanPhoto.js.ok.con_log_invariati b/api_v1/scanner/scanPhoto.js.ok.con_log_invariati new file mode 100644 index 0000000..0db35d0 --- /dev/null +++ b/api_v1/scanner/scanPhoto.js.ok.con_log_invariati @@ -0,0 +1,142 @@ +// scanner/scanPhoto.js +const path = require('path'); +const fsp = require('fs/promises'); + +const { log } = require('./logger'); +const { loadPreviousIndex, saveIndex } = require('./indexStore'); +const scanCartella = require('./scanCartella'); +const postWithAuth = require('./postWithAuth'); + +const { + WEB_ROOT, + SEND_PHOTOS, + BASE_URL, + WRITE_INDEX +} = require('../config'); + +const createCleanupFunctions = require('./orphanCleanup'); + +async function scanPhoto(dir, userName, db) { + const start = Date.now(); + log(`🔵 Inizio scan globale per user=${userName}`); + + const previousIndexTree = await loadPreviousIndex(); + const nextIndexTree = JSON.parse(JSON.stringify(previousIndexTree || {})); + + const { + buildIdsListForFolder, + removeIdFromList, + deleteThumbsById, + deleteFromDB + } = createCleanupFunctions(db); + + const photosRoot = path.resolve(__dirname, '..', '..', WEB_ROOT, 'photos'); + const userDir = path.join(photosRoot, userName, 'original'); + + let totalNew = 0; + let totalDeleted = 0; + let totalUnchanged = 0; + + let newFiles = []; + + async function scanSingleFolder(user, cartella, absCartella) { + log(`📁 Inizio cartella: ${cartella}`); + + let idsIndex = await buildIdsListForFolder(user, cartella); + + let newCount = 0; + let unchangedCount = 0; + let deletedCount = 0; + + for await (const m of scanCartella(user, cartella, absCartella, previousIndexTree)) { + const fileName = m.path.split('/').pop(); + + if (m.unchanged) { + log(` ⚪ Invariato: ${fileName}`); + idsIndex = removeIdFromList(idsIndex, m.id); + unchangedCount++; + continue; + } + + log(` 🟢 Nuovo/Modificato: ${fileName}`); + + nextIndexTree[m.user] ??= {}; + nextIndexTree[m.user][m.cartella] ??= {}; + nextIndexTree[m.user][m.cartella][m.id] = { + id: m.id, + user: m.user, + cartella: m.cartella, + path: m.path, + hash: m._indexHash + }; + + idsIndex = removeIdFromList(idsIndex, m.id); + newCount++; + newFiles.push(m); + } + + for (const orphanId of idsIndex) { + const old = previousIndexTree?.[user]?.[cartella]?.[orphanId]; + const fileName = old?.path?.split('/').pop() || orphanId; + + log(` 🔴 Eliminato: ${fileName}`); + + await deleteThumbsById(orphanId); + deleteFromDB(orphanId); + + const userTree = nextIndexTree[user]; + if (userTree?.[cartella]?.[orphanId]) { + delete userTree[cartella][orphanId]; + } + + deletedCount++; + } + + log(`📊 Fine cartella ${cartella}: invariati=${unchangedCount}, nuovi=${newCount}, cancellati=${deletedCount}`); + + totalNew += newCount; + totalDeleted += deletedCount; + totalUnchanged += unchangedCount; + } + + let entries = []; + try { + entries = await fsp.readdir(userDir, { withFileTypes: true }); + } catch { + log(`Nessuna directory per utente ${userName}`); + return []; + } + + for (const e of entries) { + if (!e.isDirectory()) continue; + const cartella = e.name; + const absCartella = path.join(userDir, cartella); + await scanSingleFolder(userName, cartella, absCartella); + } + + if (WRITE_INDEX) { + await saveIndex(nextIndexTree); + } + + if (SEND_PHOTOS && BASE_URL && newFiles.length > 0) { + for (const p of newFiles) { + const fileName = p.path.split('/').pop(); + + try { + await postWithAuth(`${BASE_URL}/photos`, p); + log(`📤 Inviato al server: ${fileName}`); + } catch (err) { + log(`Errore invio ${fileName}: ${err.message}`); + } + } + } + + const elapsed = ((Date.now() - start) / 1000).toFixed(2); + + log(`🟣 Scan COMPLETATO: nuovi=${totalNew}, cancellati=${totalDeleted}, invariati=${totalUnchanged}`); + log(`⏱ Tempo totale: ${elapsed}s`); + + return newFiles; +} + +module.exports = scanPhoto; diff --git a/api_v1/scanner/scanPhoto.js.ok.senza_log_invariati b/api_v1/scanner/scanPhoto.js.ok.senza_log_invariati new file mode 100644 index 0000000..2d93dd2 --- /dev/null +++ b/api_v1/scanner/scanPhoto.js.ok.senza_log_invariati @@ -0,0 +1,144 @@ +// scanner/scanPhoto.js +const path = require('path'); +const fsp = require('fs/promises'); + +const { log } = require('./logger'); +const { loadPreviousIndex, saveIndex } = require('./indexStore'); +const scanCartella = require('./scanCartella'); +const postWithAuth = require('./postWithAuth'); + +const { + WEB_ROOT, + SEND_PHOTOS, + BASE_URL, + WRITE_INDEX +} = require('../config'); + +const createCleanupFunctions = require('./orphanCleanup'); + +async function scanPhoto(dir, userName, db) { + const start = Date.now(); + log(`🔵 Inizio scan globale per user=${userName}`); + + const previousIndexTree = await loadPreviousIndex(); + const nextIndexTree = JSON.parse(JSON.stringify(previousIndexTree || {})); + + const { + buildIdsListForFolder, + removeIdFromList, + deleteThumbsById, + deleteFromDB + } = createCleanupFunctions(db); + + const photosRoot = path.resolve(__dirname, '..', '..', WEB_ROOT, 'photos'); + const userDir = path.join(photosRoot, userName, 'original'); + + let totalNew = 0; + let totalDeleted = 0; + let totalUnchanged = 0; + + let newFiles = []; + + async function scanSingleFolder(user, cartella, absCartella) { + log(`📁 Inizio cartella: ${cartella}`); + + let idsIndex = await buildIdsListForFolder(user, cartella); + + let newCount = 0; + let unchangedCount = 0; + let deletedCount = 0; + + for await (const m of scanCartella(user, cartella, absCartella, previousIndexTree)) { + const fileName = m.path.split('/').pop(); + + // ⚪ FILE INVARIATO → NON LOGGARE + if (m.unchanged) { + idsIndex = removeIdFromList(idsIndex, m.id); + unchangedCount++; + continue; + } + + // 🟢 FILE NUOVO O MODIFICATO + log(` 🟢 Nuovo/Modificato: ${fileName}`); + + nextIndexTree[m.user] ??= {}; + nextIndexTree[m.user][m.cartella] ??= {}; + nextIndexTree[m.user][m.cartella][m.id] = { + id: m.id, + user: m.user, + cartella: m.cartella, + path: m.path, + hash: m._indexHash + }; + + idsIndex = removeIdFromList(idsIndex, m.id); + newCount++; + newFiles.push(m); + } + + // 🔴 ORFANI + for (const orphanId of idsIndex) { + const old = previousIndexTree?.[user]?.[cartella]?.[orphanId]; + const fileName = old?.path?.split('/').pop() || orphanId; + + log(` 🔴 Eliminato: ${fileName}`); + + await deleteThumbsById(orphanId); + deleteFromDB(orphanId); + + const userTree = nextIndexTree[user]; + if (userTree?.[cartella]?.[orphanId]) { + delete userTree[cartella][orphanId]; + } + + deletedCount++; + } + + log(`📊 Fine cartella ${cartella}: invariati=${unchangedCount}, nuovi=${newCount}, cancellati=${deletedCount}`); + + totalNew += newCount; + totalDeleted += deletedCount; + totalUnchanged += unchangedCount; + } + + let entries = []; + try { + entries = await fsp.readdir(userDir, { withFileTypes: true }); + } catch { + log(`Nessuna directory per utente ${userName}`); + return []; + } + + for (const e of entries) { + if (!e.isDirectory()) continue; + const cartella = e.name; + const absCartella = path.join(userDir, cartella); + await scanSingleFolder(userName, cartella, absCartella); + } + + if (WRITE_INDEX) { + await saveIndex(nextIndexTree); + } + + if (SEND_PHOTOS && BASE_URL && newFiles.length > 0) { + for (const p of newFiles) { + const fileName = p.path.split('/').pop(); + + try { + await postWithAuth(`${BASE_URL}/photos`, p); + log(`📤 Inviato al server: ${fileName}`); + } catch (err) { + log(`Errore invio ${fileName}: ${err.message}`); + } + } + } + + const elapsed = ((Date.now() - start) / 1000).toFixed(2); + + log(`🟣 Scan COMPLETATO: nuovi=${totalNew}, cancellati=${totalDeleted}, invariati=${totalUnchanged}`); + log(`⏱ Tempo totale: ${elapsed}s`); + + return newFiles; +} + +module.exports = scanPhoto; diff --git a/api_v1/scanner/scanPhoto.js.super_ok b/api_v1/scanner/scanPhoto.js.super_ok new file mode 100644 index 0000000..05e4564 --- /dev/null +++ b/api_v1/scanner/scanPhoto.js.super_ok @@ -0,0 +1,165 @@ +// scanner/scanPhoto.js +const path = require('path'); +const fsp = require('fs/promises'); + +const { log } = require('./logger'); +const { loadPreviousIndex, saveIndex } = require('./indexStore'); +const scanCartella = require('./scanCartella'); +const postWithAuth = require('./postWithAuth'); + +const { + WEB_ROOT, + SEND_PHOTOS, + BASE_URL, + WRITE_INDEX +} = require('../config'); + +const createCleanupFunctions = require('./orphanCleanup'); + +// Variabile per log completo o ridotto +const LOG_VERBOSE = (process.env.LOG_VERBOSE === "true"); + +async function scanPhoto(dir, userName, db) { + const start = Date.now(); + log(`🔵 Inizio scan globale per user=${userName}`); + + const previousIndexTree = await loadPreviousIndex(); + const nextIndexTree = JSON.parse(JSON.stringify(previousIndexTree || {})); + + const { + buildIdsListForFolder, + removeIdFromList, + deleteThumbsById, + deleteFromDB + } = createCleanupFunctions(db); + + const photosRoot = path.resolve(__dirname, '..', '..', WEB_ROOT, 'photos'); + const userDir = path.join(photosRoot, userName, 'original'); + + let totalNew = 0; + let totalDeleted = 0; + let totalUnchanged = 0; + + let newFiles = []; + + // --------------------------------------------------------- + // SCAN DI UNA SINGOLA CARTELLA + // --------------------------------------------------------- + async function scanSingleFolder(user, cartella, absCartella) { + log(`📁 Inizio cartella: ${cartella}`); + + let idsIndex = await buildIdsListForFolder(user, cartella); + + let newCount = 0; + let unchangedCount = 0; + let deletedCount = 0; + + for await (const m of scanCartella(user, cartella, absCartella, previousIndexTree)) { + const fileName = m.path.split('/').pop(); + + // ⚪ FILE INVARIATO + if (m.unchanged) { + if (LOG_VERBOSE) { + log(` ⚪ Invariato: ${fileName}`); + } + idsIndex = removeIdFromList(idsIndex, m.id); + unchangedCount++; + continue; + } + + // 🟢 FILE NUOVO O MODIFICATO + log(` 🟢 Nuovo/Modificato: ${fileName}`); + + nextIndexTree[m.user] ??= {}; + nextIndexTree[m.user][m.cartella] ??= {}; + nextIndexTree[m.user][m.cartella][m.id] = { + id: m.id, + user: m.user, + cartella: m.cartella, + path: m.path, + hash: m._indexHash + }; + + idsIndex = removeIdFromList(idsIndex, m.id); + newCount++; + newFiles.push(m); + } + + // 🔴 ORFANI (FILE CANCELLATI) + for (const orphanId of idsIndex) { + const old = previousIndexTree?.[user]?.[cartella]?.[orphanId]; + const fileName = old?.path?.split('/').pop() || orphanId; + + log(` 🔴 Eliminato: ${fileName}`); + + await deleteThumbsById(orphanId); + deleteFromDB(orphanId); + + const userTree = nextIndexTree[user]; + if (userTree?.[cartella]?.[orphanId]) { + delete userTree[cartella][orphanId]; + } + + deletedCount++; + } + + log(`📊 Fine cartella ${cartella}: invariati=${unchangedCount}, nuovi=${newCount}, cancellati=${deletedCount}`); + + totalNew += newCount; + totalDeleted += deletedCount; + totalUnchanged += unchangedCount; + } + + // --------------------------------------------------------- + // SCAN SOTTOCARTELLE + // --------------------------------------------------------- + let entries = []; + try { + entries = await fsp.readdir(userDir, { withFileTypes: true }); + } catch { + log(`Nessuna directory per utente ${userName}`); + return []; + } + + for (const e of entries) { + if (!e.isDirectory()) continue; + const cartella = e.name; + const absCartella = path.join(userDir, cartella); + await scanSingleFolder(userName, cartella, absCartella); + } + + // --------------------------------------------------------- + // SALVO INDEX + // --------------------------------------------------------- + if (WRITE_INDEX) { + await saveIndex(nextIndexTree); + } + + // --------------------------------------------------------- + // INVIO AL SERVER / POPOLAZIONE DB + // --------------------------------------------------------- + if (SEND_PHOTOS && BASE_URL && newFiles.length > 0) { + for (const p of newFiles) { + const fileName = p.path.split('/').pop(); + + try { + await postWithAuth(`${BASE_URL}/photos`, p); + log(`📤 Inviato al server: ${fileName}`); + } catch (err) { + log(`Errore invio ${fileName}: ${err.message}`); + } + } + } + + // --------------------------------------------------------- + // FINE SCAN + // --------------------------------------------------------- + const elapsed = ((Date.now() - start) / 1000).toFixed(2); + + log(`🟣 Scan COMPLETATO: nuovi=${totalNew}, cancellati=${totalDeleted}, invariati=${totalUnchanged}`); + log(`⏱ Tempo totale: ${elapsed}s`); + + return newFiles; +} + +module.exports = scanPhoto; diff --git a/api_v1/scanner/scanPhoto.js.super_ok2 b/api_v1/scanner/scanPhoto.js.super_ok2 new file mode 100644 index 0000000..db8f7fe --- /dev/null +++ b/api_v1/scanner/scanPhoto.js.super_ok2 @@ -0,0 +1,192 @@ +// scanner/scanPhoto.js +const path = require('path'); +const fsp = require('fs/promises'); + +const { log } = require('./logger'); +const { loadPreviousIndex, saveIndex } = require('./indexStore'); +const scanCartella = require('./scanCartella'); +const postWithAuth = require('./postWithAuth'); + +const { + WEB_ROOT, + SEND_PHOTOS, + BASE_URL, + WRITE_INDEX +} = require('../config'); + +const createCleanupFunctions = require('./orphanCleanup'); + +// Variabile per log completo o ridotto +const LOG_VERBOSE = (process.env.LOG_VERBOSE === "true"); + +async function scanPhoto(dir, userName, db) { + const start = Date.now(); + log(`🔵 Inizio scan globale per user=${userName}`); + + const previousIndexTree = await loadPreviousIndex(); + const nextIndexTree = JSON.parse(JSON.stringify(previousIndexTree || {})); + + const { + buildIdsListForFolder, + removeIdFromList, + deleteThumbsById, + deleteFromDB + } = createCleanupFunctions(db); + + const photosRoot = path.resolve(__dirname, '..', '..', WEB_ROOT, 'photos'); + const userDir = path.join(photosRoot, userName, 'original'); + + let totalNew = 0; + let totalDeleted = 0; + let totalUnchanged = 0; + + let newFiles = []; + + // --------------------------------------------------------- + // 1) CALCOLO TOTALE FILE (contatore globale) + // --------------------------------------------------------- + let TOTAL_FILES = 0; + + let entries = []; + try { + entries = await fsp.readdir(userDir, { withFileTypes: true }); + } catch { + log(`Nessuna directory per utente ${userName}`); + return []; + } + + for (const e of entries) { + if (!e.isDirectory()) continue; + const cartella = e.name; + const absCartella = path.join(userDir, cartella); + + for await (const _ of scanCartella(userName, cartella, absCartella, previousIndexTree)) { + TOTAL_FILES++; + } + } + + if (TOTAL_FILES === 0) { + log(`Nessun file trovato per user=${userName}`); + return []; + } + + log(`📦 File totali da processare: ${TOTAL_FILES}`); + + // contatore progressivo + let CURRENT = 0; + + // --------------------------------------------------------- + // 2) SCAN REALE DELLE CARTELLE + // --------------------------------------------------------- + async function scanSingleFolder(user, cartella, absCartella) { + log(`📁 Inizio cartella: ${cartella}`); + + let idsIndex = await buildIdsListForFolder(user, cartella); + + let newCount = 0; + let unchangedCount = 0; + let deletedCount = 0; + + for await (const m of scanCartella(user, cartella, absCartella, previousIndexTree)) { + CURRENT++; + const fileName = m.path.split('/').pop(); + const prefix = `[${CURRENT}/${TOTAL_FILES}]`; + + // ⚪ FILE INVARIATO + if (m.unchanged) { + if (LOG_VERBOSE) { + log(`${prefix} ⚪ Invariato: ${fileName}`); + } + idsIndex = removeIdFromList(idsIndex, m.id); + unchangedCount++; + continue; + } + + // 🟢 FILE NUOVO O MODIFICATO + log(`${prefix} 🟢 Nuovo/Modificato: ${fileName}`); + + nextIndexTree[m.user] ??= {}; + nextIndexTree[m.user][m.cartella] ??= {}; + nextIndexTree[m.user][m.cartella][m.id] = { + id: m.id, + user: m.user, + cartella: m.cartella, + path: m.path, + hash: m._indexHash + }; + + idsIndex = removeIdFromList(idsIndex, m.id); + newCount++; + newFiles.push(m); + } + + // 🔴 ORFANI + for (const orphanId of idsIndex) { + const old = previousIndexTree?.[user]?.[cartella]?.[orphanId]; + const fileName = old?.path?.split('/').pop() || orphanId; + + log(` 🔴 Eliminato: ${fileName}`); + + await deleteThumbsById(orphanId); + deleteFromDB(orphanId); + + const userTree = nextIndexTree[user]; + if (userTree?.[cartella]?.[orphanId]) { + delete userTree[cartella][orphanId]; + } + + deletedCount++; + } + + log(`📊 Fine cartella ${cartella}: invariati=${unchangedCount}, nuovi=${newCount}, cancellati=${deletedCount}`); + + totalNew += newCount; + totalDeleted += deletedCount; + totalUnchanged += unchangedCount; + } + + // --------------------------------------------------------- + // SCAN SOTTOCARTELLE (REPLAY) + // --------------------------------------------------------- + for (const e of entries) { + if (!e.isDirectory()) continue; + const cartella = e.name; + const absCartella = path.join(userDir, cartella); + await scanSingleFolder(userName, cartella, absCartella); + } + + // --------------------------------------------------------- + // SALVO INDEX + // --------------------------------------------------------- + if (WRITE_INDEX) { + await saveIndex(nextIndexTree); + } + + // --------------------------------------------------------- + // INVIO AL SERVER / POPOLAZIONE DB + // --------------------------------------------------------- + if (SEND_PHOTOS && BASE_URL && newFiles.length > 0) { + for (const p of newFiles) { + const fileName = p.path.split('/').pop(); + + try { + await postWithAuth(`${BASE_URL}/photos`, p); + log(`📤 Inviato al server: ${fileName}`); + } catch (err) { + log(`Errore invio ${fileName}: ${err.message}`); + } + } + } + + // --------------------------------------------------------- + // FINE SCAN + // --------------------------------------------------------- + const elapsed = ((Date.now() - start) / 1000).toFixed(2); + + log(`🟣 Scan COMPLETATO: nuovi=${totalNew}, cancellati=${totalDeleted}, invariati=${totalUnchanged}`); + log(`⏱ Tempo totale: ${elapsed}s`); + + return newFiles; +} + +module.exports = scanPhoto; diff --git a/api_v1/scanner/scanUser.js b/api_v1/scanner/scanUser.js index b1afbe9..1c28ec2 100644 --- a/api_v1/scanner/scanUser.js +++ b/api_v1/scanner/scanUser.js @@ -7,12 +7,12 @@ const { sha256 } = require('./utils'); const { SUPPORTED_EXTS } = require('../config'); async function scanUserRoot(userName, userDir, previousIndexTree) { - console.log(`[SCAN] scanUserRoot → user=${userName}, dir=${userDir}`); + console.log(`\n🔵 Inizio scan user: ${userName}`); const results = []; const entries = await fsp.readdir(userDir, { withFileTypes: true }); - // 1) File nella root + // ROOT FILES for (const e of entries) { if (e.isDirectory()) continue; @@ -29,14 +29,7 @@ async function scanUserRoot(userName, userDir, previousIndexTree) { const hash = sha256(`${st.size}-${st.mtimeMs}`); const prev = previousIndexTree?.[userName]?.[cartella]?.[id]; - console.log(`[SCAN] ROOT CHECK → ${fileRelPath}, hash=${hash}, prevHash=${prev?.hash}`); - - if (prev && prev.hash === hash) { - console.log(`[SCAN] SKIP ROOT (unchanged) → ${fileRelPath}`); - continue; - } - - console.log(`[SCAN] PROCESSING ROOT → ${fileRelPath}`); + if (prev && prev.hash === hash) continue; const meta = await processFile( userName, @@ -54,14 +47,14 @@ async function scanUserRoot(userName, userDir, previousIndexTree) { results.push(meta); } - // 2) Sottocartelle + // SUBFOLDERS for (const e of entries) { if (!e.isDirectory()) continue; const cartella = e.name; const absCartella = path.join(userDir, cartella); - console.log(`[SCAN] → Scansiono cartella: ${cartella}`); + console.log(` 📁 Cartella: ${cartella}`); const files = await scanCartella(userName, cartella, absCartella, previousIndexTree); results.push(...files); diff --git a/api_v1/scanner/scanUser.js.ok b/api_v1/scanner/scanUser.js.ok deleted file mode 100644 index b1afbe9..0000000 --- a/api_v1/scanner/scanUser.js.ok +++ /dev/null @@ -1,73 +0,0 @@ -// scanner/scanUser.js -const path = require('path'); -const fsp = require('fs/promises'); -const scanCartella = require('./scanCartella'); -const processFile = require('./processFile'); -const { sha256 } = require('./utils'); -const { SUPPORTED_EXTS } = require('../config'); - -async function scanUserRoot(userName, userDir, previousIndexTree) { - console.log(`[SCAN] scanUserRoot → user=${userName}, dir=${userDir}`); - - const results = []; - const entries = await fsp.readdir(userDir, { withFileTypes: true }); - - // 1) File nella root - for (const e of entries) { - if (e.isDirectory()) continue; - - const ext = path.extname(e.name).toLowerCase(); - if (!SUPPORTED_EXTS.has(ext)) continue; - - const absPath = path.join(userDir, e.name); - const st = await fsp.stat(absPath); - - const fileRelPath = e.name; - const cartella = '_root'; - const id = sha256(`${userName}/${cartella}/${fileRelPath}`); - - const hash = sha256(`${st.size}-${st.mtimeMs}`); - const prev = previousIndexTree?.[userName]?.[cartella]?.[id]; - - console.log(`[SCAN] ROOT CHECK → ${fileRelPath}, hash=${hash}, prevHash=${prev?.hash}`); - - if (prev && prev.hash === hash) { - console.log(`[SCAN] SKIP ROOT (unchanged) → ${fileRelPath}`); - continue; - } - - console.log(`[SCAN] PROCESSING ROOT → ${fileRelPath}`); - - const meta = await processFile( - userName, - cartella, - fileRelPath, - absPath, - ext, - st - ); - - meta.id = id; - meta.path = `/photos/${userName}/original/${cartella}/${fileRelPath}`; - meta._indexHash = hash; - - results.push(meta); - } - - // 2) Sottocartelle - for (const e of entries) { - if (!e.isDirectory()) continue; - - const cartella = e.name; - const absCartella = path.join(userDir, cartella); - - console.log(`[SCAN] → Scansiono cartella: ${cartella}`); - - const files = await scanCartella(userName, cartella, absCartella, previousIndexTree); - results.push(...files); - } - - return results; -} - -module.exports = scanUserRoot; diff --git a/api_v1/scanner/t.js b/api_v1/scanner/t.js deleted file mode 100644 index fb7fdd3..0000000 --- a/api_v1/scanner/t.js +++ /dev/null @@ -1,36 +0,0 @@ -// idsIndex-test.js - -// Questa è la stessa funzione che usi nel tuo programma -function removeIdFromList(idsIndex, id) { - return idsIndex.filter(x => x !== id); -} - -// Simuliamo idsIndex come lo costruisce il tuo programma: -// un array di stringhe (ID) -const idsIndex = [ - "b3cda38a2fdee7ba9bdd2ea07d936b2fa5361baa21be2b2c29f1ad8ebcc4c31e", - "09ef919ff2a105ad75d2e7631b9c02228c1784df5048c98276fd379f1533360b", - "dba1de0e187650e62118279beb2d83e8759a6d4fb8a5412a6a18cdc4e01a33d7" -]; - -// Prendiamo il primo ID -const idToRemove = idsIndex[0]; - -// LOG iniziali -console.log("=== TEST RIMOZIONE ID ==="); -console.log("idsIndex iniziale:", idsIndex); -console.log("ID da rimuovere:", idToRemove); -console.log("--------------------------"); - -// Rimozione -const newList = removeIdFromList(idsIndex, idToRemove); - -// LOG finali -console.log("idsIndex dopo removeIdFromList:", newList); - -// Verifica manuale -if (newList.includes(idToRemove)) { - console.log("❌ ERRORE: l'ID NON è stato rimosso!"); -} else { - console.log("✅ OK: l'ID è stato rimosso correttamente."); -} diff --git a/public/admin.html b/public/admin.html index 2590c44..44bc399 100644 --- a/public/admin.html +++ b/public/admin.html @@ -3,7 +3,32 @@