ruoxi_sun
58657935fc
Vendor LLMmap / llm-verify / llm-fingerprint-detector under bash/fingerprint/tools so the three fingerprint benchmarks run with only /data1/eval mounted (no /data1/xii dependency): - run.py DEFAULT_TOOLS_ROOT prefers builtin tools/, falls back to /data1/xii - exclude .git / node_modules / template backups - detector dist/ (pre-built) retained; node_modules not needed at runtime
57 lines
1.8 KiB
Python
57 lines
1.8 KiB
Python
import sys
|
|
import os
|
|
import json
|
|
import argparse
|
|
|
|
from LLMmap.dataset import load_datasets
|
|
from LLMmap.inference import load_LLMmap, write_templates
|
|
from LLMmap.templates import template_generation
|
|
from LLMmap import TEMPLATE_NAME
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Generate and export LLMs templates for open LLMmap inference model based on training set.")
|
|
parser.add_argument("model_home_dir", type=str, help="Path to the model home directory")
|
|
args = parser.parse_args()
|
|
|
|
model_home_dir = args.model_home_dir
|
|
conf, inf = load_LLMmap(model_home_dir, device='cpu')
|
|
|
|
if not conf['is_open']:
|
|
print("Applicable to only open-set inference model. Aborting...")
|
|
sys.exit(1)
|
|
|
|
siamese = False
|
|
(loader_train, loader_test), cache, (dataset_train, dataset_test) = load_datasets(
|
|
conf,
|
|
siamese=siamese,
|
|
ks=conf.get('num_istances_dataset', None)
|
|
)
|
|
|
|
results = template_generation(inf.model, loader_train, loader_test)
|
|
print(f"Accuracy on test set: {results['accuracy']}")
|
|
|
|
templates_map = {}
|
|
templates = results['templates']
|
|
for i in range(len(templates)):
|
|
llm = inf.label_map[i]
|
|
templates_map[llm] = templates[i]
|
|
|
|
template_out = os.path.join(model_home_dir, TEMPLATE_NAME)
|
|
|
|
if os.path.exists(template_out):
|
|
confirm = input(f"'{template_out}' already exists. Overwrite? (y/n): ")
|
|
if confirm.lower() != 'y':
|
|
print("Aborting.")
|
|
sys.exit(1)
|
|
|
|
write_templates(template_out, templates_map)
|
|
|
|
print(f"Templates saved to '{template_out}'")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description="Create templates for a pre-trained LLMmap open inference model.")
|
|
parser.add_argument("model_home_dir", type=str, help="Path to the model home directory")
|
|
args = parser.parse_args()
|
|
main() |