543 lines
9.8 KiB
JavaScript
543 lines
9.8 KiB
JavaScript
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
const db = require('../db/knex');
|
|
|
|
const {
|
|
runFaceKnn,
|
|
} = require('./faceKnnRuntime');
|
|
|
|
function parseArguments(argv) {
|
|
const experimentIndex =
|
|
argv.indexOf('--experiment');
|
|
|
|
const outputIndex =
|
|
argv.indexOf('--output');
|
|
|
|
const backendIndex =
|
|
argv.indexOf('--backend');
|
|
|
|
return {
|
|
experimentPath:
|
|
experimentIndex >= 0
|
|
? path.resolve(
|
|
String(
|
|
argv[experimentIndex + 1]
|
|
|| '',
|
|
),
|
|
)
|
|
: '',
|
|
|
|
outputPath:
|
|
outputIndex >= 0
|
|
? path.resolve(
|
|
String(
|
|
argv[outputIndex + 1]
|
|
|| '',
|
|
),
|
|
)
|
|
: '',
|
|
|
|
backend:
|
|
backendIndex >= 0
|
|
? String(
|
|
argv[backendIndex + 1]
|
|
|| '',
|
|
)
|
|
: '',
|
|
|
|
keepTemporaryFiles:
|
|
argv.includes(
|
|
'--keep-temporary-files',
|
|
),
|
|
};
|
|
}
|
|
|
|
function mean(values) {
|
|
if (values.length === 0) {
|
|
return null;
|
|
}
|
|
|
|
return (
|
|
values.reduce(
|
|
(sum, value) =>
|
|
sum + value,
|
|
0,
|
|
)
|
|
/ values.length
|
|
);
|
|
}
|
|
|
|
function compareResults({
|
|
experiment,
|
|
runtime,
|
|
}) {
|
|
const pythonQueries =
|
|
experiment
|
|
?.evaluations
|
|
?.greedy_gain
|
|
?.queries;
|
|
|
|
if (
|
|
!Array.isArray(pythonQueries)
|
|
|| pythonQueries.length === 0
|
|
) {
|
|
throw new Error(
|
|
'Query greedy_gain mancanti',
|
|
);
|
|
}
|
|
|
|
const openclResults =
|
|
runtime.output.results;
|
|
|
|
if (
|
|
!Array.isArray(openclResults)
|
|
|| openclResults.length
|
|
!== pythonQueries.length
|
|
) {
|
|
throw new Error(
|
|
'Numero risultati face_knn incoerente',
|
|
);
|
|
}
|
|
|
|
const threshold =
|
|
Number(
|
|
runtime.config.recognitionThreshold,
|
|
);
|
|
|
|
const tolerance =
|
|
Number(
|
|
runtime.config.scoreTolerance,
|
|
);
|
|
|
|
let top1Matches = 0;
|
|
let decisionMatches = 0;
|
|
let bestScoreMatches = 0;
|
|
let top3MeanMatches = 0;
|
|
let maximumScoreError = 0;
|
|
let maximumTop3MeanError = 0;
|
|
|
|
const comparisons = [];
|
|
const mismatches = [];
|
|
|
|
for (
|
|
let queryIndex = 0;
|
|
queryIndex < pythonQueries.length;
|
|
queryIndex += 1
|
|
) {
|
|
const pythonQuery =
|
|
pythonQueries[queryIndex];
|
|
|
|
const nativeQuery =
|
|
openclResults[queryIndex];
|
|
|
|
const queryMetadata =
|
|
runtime.queryRows[queryIndex];
|
|
|
|
if (
|
|
pythonQuery.face_id
|
|
!== queryMetadata.face_id
|
|
) {
|
|
throw new Error(
|
|
'Ordine query incoerente '
|
|
+ `all'indice ${queryIndex}`,
|
|
);
|
|
}
|
|
|
|
const nativeBestIndex =
|
|
Number(nativeQuery.best_index);
|
|
|
|
const nativePrototype =
|
|
runtime.prototypeRows[
|
|
nativeBestIndex
|
|
];
|
|
|
|
if (!nativePrototype) {
|
|
throw new Error(
|
|
'Indice prototype OpenCL '
|
|
+ `non valido: ${nativeBestIndex}`,
|
|
);
|
|
}
|
|
|
|
const pythonBestScore =
|
|
pythonQuery.best_similarity == null
|
|
? null
|
|
: Number(
|
|
pythonQuery.best_similarity,
|
|
);
|
|
|
|
const nativeBestScore =
|
|
Number(
|
|
nativeQuery.best_score,
|
|
);
|
|
|
|
const pythonTop3Mean =
|
|
pythonQuery.top3_mean == null
|
|
? null
|
|
: Number(
|
|
pythonQuery.top3_mean,
|
|
);
|
|
|
|
const nativeTop3Mean =
|
|
mean(
|
|
(nativeQuery.neighbors || [])
|
|
.slice(0, 3)
|
|
.map((neighbor) =>
|
|
Number(neighbor.score)),
|
|
);
|
|
|
|
const pythonRecognized =
|
|
pythonBestScore != null
|
|
&& Number.isFinite(
|
|
pythonBestScore,
|
|
)
|
|
&& pythonBestScore >= threshold;
|
|
|
|
const nativeRecognized =
|
|
Boolean(nativeQuery.recognized);
|
|
|
|
const prototypeMatch =
|
|
String(
|
|
pythonQuery
|
|
.best_prototype_face_id
|
|
|| '',
|
|
)
|
|
=== nativePrototype.face_id;
|
|
|
|
const decisionMatch =
|
|
pythonRecognized
|
|
=== nativeRecognized;
|
|
|
|
const scoreError =
|
|
pythonBestScore == null
|
|
? null
|
|
: Math.abs(
|
|
pythonBestScore
|
|
- nativeBestScore,
|
|
);
|
|
|
|
const top3MeanError =
|
|
(
|
|
pythonTop3Mean == null
|
|
|| nativeTop3Mean == null
|
|
)
|
|
? null
|
|
: Math.abs(
|
|
pythonTop3Mean
|
|
- nativeTop3Mean,
|
|
);
|
|
|
|
const scoreMatch =
|
|
scoreError != null
|
|
&& scoreError <= tolerance;
|
|
|
|
const top3MeanMatch =
|
|
top3MeanError != null
|
|
&& top3MeanError <= tolerance;
|
|
|
|
if (prototypeMatch) {
|
|
top1Matches += 1;
|
|
}
|
|
|
|
if (decisionMatch) {
|
|
decisionMatches += 1;
|
|
}
|
|
|
|
if (scoreMatch) {
|
|
bestScoreMatches += 1;
|
|
}
|
|
|
|
if (top3MeanMatch) {
|
|
top3MeanMatches += 1;
|
|
}
|
|
|
|
if (scoreError != null) {
|
|
maximumScoreError =
|
|
Math.max(
|
|
maximumScoreError,
|
|
scoreError,
|
|
);
|
|
}
|
|
|
|
if (top3MeanError != null) {
|
|
maximumTop3MeanError =
|
|
Math.max(
|
|
maximumTop3MeanError,
|
|
top3MeanError,
|
|
);
|
|
}
|
|
|
|
const comparison = {
|
|
query_index:
|
|
queryIndex,
|
|
|
|
face_id:
|
|
pythonQuery.face_id,
|
|
|
|
photo_name:
|
|
pythonQuery.photo_name,
|
|
|
|
is_prototype:
|
|
Boolean(
|
|
pythonQuery.is_prototype,
|
|
),
|
|
|
|
excluded_index:
|
|
nativeQuery.excluded_index,
|
|
|
|
python_best_prototype_face_id:
|
|
pythonQuery
|
|
.best_prototype_face_id
|
|
|| null,
|
|
|
|
native_best_prototype_face_id:
|
|
nativePrototype.face_id,
|
|
|
|
native_best_prototype_index:
|
|
nativeBestIndex,
|
|
|
|
python_best_similarity:
|
|
pythonBestScore,
|
|
|
|
native_best_similarity:
|
|
nativeBestScore,
|
|
|
|
absolute_score_error:
|
|
scoreError,
|
|
|
|
python_top3_mean:
|
|
pythonTop3Mean,
|
|
|
|
native_top3_mean:
|
|
nativeTop3Mean,
|
|
|
|
absolute_top3_mean_error:
|
|
top3MeanError,
|
|
|
|
python_recognized:
|
|
pythonRecognized,
|
|
|
|
native_recognized:
|
|
nativeRecognized,
|
|
|
|
prototype_match:
|
|
prototypeMatch,
|
|
|
|
decision_match:
|
|
decisionMatch,
|
|
|
|
score_match:
|
|
scoreMatch,
|
|
|
|
top3_mean_match:
|
|
top3MeanMatch,
|
|
};
|
|
|
|
comparisons.push(comparison);
|
|
|
|
if (
|
|
!prototypeMatch
|
|
|| !decisionMatch
|
|
|| !scoreMatch
|
|
|| !top3MeanMatch
|
|
) {
|
|
mismatches.push(comparison);
|
|
}
|
|
}
|
|
|
|
return {
|
|
threshold,
|
|
tolerance,
|
|
query_count:
|
|
pythonQueries.length,
|
|
prototype_count:
|
|
runtime.prototypeRows.length,
|
|
top1_matches:
|
|
top1Matches,
|
|
decision_matches:
|
|
decisionMatches,
|
|
best_score_matches:
|
|
bestScoreMatches,
|
|
top3_mean_matches:
|
|
top3MeanMatches,
|
|
maximum_score_error:
|
|
maximumScoreError,
|
|
maximum_top3_mean_error:
|
|
maximumTop3MeanError,
|
|
mismatch_count:
|
|
mismatches.length,
|
|
mismatches,
|
|
comparisons,
|
|
};
|
|
}
|
|
|
|
async function main() {
|
|
const options =
|
|
parseArguments(
|
|
process.argv.slice(2),
|
|
);
|
|
|
|
if (!options.experimentPath) {
|
|
throw new Error(
|
|
'Uso: node tools/runFaceKnnShadow.js '
|
|
+ '--experiment FILE '
|
|
+ '[--output FILE] '
|
|
+ '[--backend opencl|cpu]',
|
|
);
|
|
}
|
|
|
|
const experiment =
|
|
JSON.parse(
|
|
fs.readFileSync(
|
|
options.experimentPath,
|
|
'utf8',
|
|
),
|
|
);
|
|
|
|
const person =
|
|
experiment.person || {};
|
|
|
|
const pythonQueries =
|
|
experiment
|
|
?.evaluations
|
|
?.greedy_gain
|
|
?.queries;
|
|
|
|
if (
|
|
!person.id
|
|
|| !person.user
|
|
|| !Array.isArray(pythonQueries)
|
|
) {
|
|
throw new Error(
|
|
'Esperimento incompleto',
|
|
);
|
|
}
|
|
|
|
const runtime =
|
|
await runFaceKnn({
|
|
connection: db,
|
|
personId: person.id,
|
|
user: person.user,
|
|
queryFaceIds:
|
|
pythonQueries.map(
|
|
(query) => query.face_id,
|
|
),
|
|
backend:
|
|
options.backend || undefined,
|
|
keepTemporaryFiles:
|
|
options.keepTemporaryFiles,
|
|
});
|
|
|
|
const comparison =
|
|
compareResults({
|
|
experiment,
|
|
runtime,
|
|
});
|
|
|
|
const report = {
|
|
mode: 'shadow',
|
|
generated_at:
|
|
new Date().toISOString(),
|
|
experiment_path:
|
|
options.experimentPath,
|
|
database:
|
|
experiment.database || null,
|
|
person: {
|
|
id: person.id,
|
|
user: person.user,
|
|
name: person.name || null,
|
|
},
|
|
backend:
|
|
runtime.output.backend,
|
|
device_name:
|
|
runtime.output.device_name,
|
|
elapsed_ms:
|
|
runtime.output.elapsed_ms,
|
|
query_count:
|
|
comparison.query_count,
|
|
prototype_count:
|
|
comparison.prototype_count,
|
|
threshold:
|
|
comparison.threshold,
|
|
tolerance:
|
|
comparison.tolerance,
|
|
top1_matches:
|
|
comparison.top1_matches,
|
|
decision_matches:
|
|
comparison.decision_matches,
|
|
best_score_matches:
|
|
comparison.best_score_matches,
|
|
top3_mean_matches:
|
|
comparison.top3_mean_matches,
|
|
maximum_score_error:
|
|
comparison.maximum_score_error,
|
|
maximum_top3_mean_error:
|
|
comparison.maximum_top3_mean_error,
|
|
mismatch_count:
|
|
comparison.mismatch_count,
|
|
mismatches:
|
|
comparison.mismatches,
|
|
comparisons:
|
|
comparison.comparisons,
|
|
prototypes:
|
|
runtime.prototypeRows,
|
|
native_stderr:
|
|
runtime.stderr,
|
|
temporary_directory:
|
|
runtime.temporaryDirectory,
|
|
};
|
|
|
|
const reportText =
|
|
`${JSON.stringify(
|
|
report,
|
|
null,
|
|
2,
|
|
)}\n`;
|
|
|
|
if (options.outputPath) {
|
|
fs.mkdirSync(
|
|
path.dirname(
|
|
options.outputPath,
|
|
),
|
|
{
|
|
recursive: true,
|
|
},
|
|
);
|
|
|
|
const temporaryOutput =
|
|
`${options.outputPath}.tmp-`
|
|
+ process.pid;
|
|
|
|
fs.writeFileSync(
|
|
temporaryOutput,
|
|
reportText,
|
|
'utf8',
|
|
);
|
|
|
|
fs.renameSync(
|
|
temporaryOutput,
|
|
options.outputPath,
|
|
);
|
|
}
|
|
|
|
process.stdout.write(reportText);
|
|
|
|
if (comparison.mismatch_count > 0) {
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
main()
|
|
.catch((error) => {
|
|
console.error(
|
|
'[faceKnnShadow] errore:',
|
|
error?.stack
|
|
|| error?.message
|
|
|| error,
|
|
);
|
|
|
|
process.exitCode = 1;
|
|
})
|
|
.finally(async () => {
|
|
await db.destroy();
|
|
});
|