Skip to content

R
RAGRights
Commit 2218e5d6 authored by Delvallez Delvallez's avatar Delvallez Delvallez
Browse files
Options

slides au 2/06 soir

parent 0864687c
Show whitespace changes
Inline Side-by-side
Showing with 714 additions and 300 deletions
.gitignore
View file @ 2218e5d6
...@@ -6,8 +6,14 @@ ...@@ -6,8 +6,14 @@
*.snm *.snm
*.synctex.gz *.synctex.gz
*.toc *.toc
*_files/ #*_files/
# Quarto # Quarto
# .html # .html
# *_files # *_files
**/.jupyter_cache/
*.quarto_ipynb_*
# Document à cacher
seminaireCA/indications_oral.md
seminaireCA/test/
\ No newline at end of file
seminaireCA/biblio.bib
View file @ 2218e5d6
...@@ -130,3 +130,58 @@ ...@@ -130,3 +130,58 @@
note = {arXiv:2302.13971}, note = {arXiv:2302.13971},
keywords = {Computer Science - Computation and Language}, keywords = {Computer Science - Computation and Language},
} }
@misc{nanda_transformerlens_2022,
title = {{TransformerLens}},
url = {https://github.com/TransformerLensOrg/TransformerLens},
author = {Nanda, Neel and Bloom, Joseph},
year = {2022},
}
@article{somvanshi_bridging_2026,
title = {Bridging the {Black} {Box}: {A} {Survey} on {Mechanistic} {Interpretability} in {AI}},
volume = {58},
issn = {0360-0300, 1557-7341},
shorttitle = {Bridging the {Black} {Box}},
url = {https://dl.acm.org/doi/10.1145/3787104},
doi = {10.1145/3787104},
abstract = {Mechanistic interpretability seeks to reverse-engineer the internal logic of neural networks by uncovering human-understandable circuits, algorithms, and causal structures that drive model behavior. Unlike post hoc explanations that describe what models do, this paradigm focuses on why and how they compute, tracing information flow through neurons, attention heads, and activation pathways. This survey provides a high-level synthesis of the field-highlighting its motivation, conceptual foundations, and methodological taxonomy rather than enumerating individual techniques. We organize mechanistic interpretability across three abstraction layers—
neurons
,
circuits
, and
algorithms
—and three evaluation perspectives:
behavioral
,
counterfactual
, and
causal
. We further discuss representative approaches and toolchains that enable structural analysis of modern AI systems, outlining how mechanistic interpretability bridges theoretical insights with practical transparency. Despite rapid progress, challenges persist in scaling these analyses to frontier models, resolving polysemantic representations, and establishing standardized causal benchmarks. By connecting historical evolution, current methodologies, and emerging research directions, this survey aims to provide an integrative framework for understanding how mechanistic interpretability can support transparency, reliability, and governance in large-scale AI.},
language = {en},
number = {8},
urldate = {2026-06-01},
journal = {ACM Computing Surveys},
author = {Somvanshi, Shriyank and Islam, Md Monzurul and Rafe, Amir and Tusti, Anannya Ghosh and Chakraborty, Arka and Baitullah, Anika and Chowdhury, Tausif Islam and Alnawmasi, Nawaf and Dutta, Anandi and Das, Subasish},
month = jun,
year = {2026},
pages = {1--35},
}
@inproceedings{wang_minilmv2_2021,
address = {Online},
title = {{MiniLMv2}: {Multi}-{Head} {Self}-{Attention} {Relation} {Distillation} for {Compressing} {Pretrained} {Transformers}},
shorttitle = {{MiniLMv2}},
url = {https://aclanthology.org/2021.findings-acl.188/},
doi = {10.18653/v1/2021.findings-acl.188},
urldate = {2026-06-02},
booktitle = {Findings of the {Association} for {Computational} {Linguistics}: {ACL}-{IJCNLP} 2021},
publisher = {Association for Computational Linguistics},
author = {Wang, Wenhui and Bao, Hangbo and Huang, Shaohan and Dong, Li and Wei, Furu},
editor = {Zong, Chengqing and Xia, Fei and Li, Wenjie and Navigli, Roberto},
month = aug,
year = {2021},
pages = {2140--2151},
}
\ No newline at end of file
seminaireCA/slides.html
View file @ 2218e5d6
...@@ -31,72 +31,6 @@ ...@@ -31,72 +31,6 @@
margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */ margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
vertical-align: middle; vertical-align: middle;
} }
/* CSS for syntax highlighting */
html { -webkit-text-size-adjust: 100%; }
pre > code.sourceCode { white-space: pre; position: relative; }
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
pre > code.sourceCode > span:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode > span { color: inherit; text-decoration: inherit; }
div.sourceCode { margin: 1em 0; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
pre > code.sourceCode { white-space: pre-wrap; }
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
}
pre.numberSource code
{ counter-reset: source-line 0; }
pre.numberSource code > span
{ position: relative; left: -4em; counter-increment: source-line; }
pre.numberSource code > span > a:first-child::before
{ content: counter(source-line);
position: relative; left: -1em; text-align: right; vertical-align: baseline;
border: none; display: inline-block;
-webkit-touch-callout: none; -webkit-user-select: none;
-khtml-user-select: none; -moz-user-select: none;
-ms-user-select: none; user-select: none;
padding: 0 4px; width: 4em;
color: #aaaaaa;
}
pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; }
div.sourceCode
{ color: #003b4f; background-color: #f1f3f5; }
@media screen {
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
}
code span { color: #003b4f; } /* Normal */
code span.al { color: #ad0000; } /* Alert */
code span.an { color: #5e5e5e; } /* Annotation */
code span.at { color: #657422; } /* Attribute */
code span.bn { color: #ad0000; } /* BaseN */
code span.bu { } /* BuiltIn */
code span.cf { color: #003b4f; font-weight: bold; } /* ControlFlow */
code span.ch { color: #20794d; } /* Char */
code span.cn { color: #8f5902; } /* Constant */
code span.co { color: #5e5e5e; } /* Comment */
code span.cv { color: #5e5e5e; font-style: italic; } /* CommentVar */
code span.do { color: #5e5e5e; font-style: italic; } /* Documentation */
code span.dt { color: #ad0000; } /* DataType */
code span.dv { color: #ad0000; } /* DecVal */
code span.er { color: #ad0000; } /* Error */
code span.ex { } /* Extension */
code span.fl { color: #ad0000; } /* Float */
code span.fu { color: #4758ab; } /* Function */
code span.im { color: #00769e; } /* Import */
code span.in { color: #5e5e5e; } /* Information */
code span.kw { color: #003b4f; font-weight: bold; } /* Keyword */
code span.op { color: #5e5e5e; } /* Operator */
code span.ot { color: #003b4f; } /* Other */
code span.pp { color: #ad0000; } /* Preprocessor */
code span.sc { color: #5e5e5e; } /* SpecialChar */
code span.ss { color: #20794d; } /* SpecialString */
code span.st { color: #20794d; } /* String */
code span.va { color: #111111; } /* Variable */
code span.vs { color: #20794d; } /* VerbatimString */
code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */
/* CSS for citations */ /* CSS for citations */
div.csl-bib-body { } div.csl-bib-body { }
div.csl-entry { div.csl-entry {
...@@ -118,6 +52,7 @@ ...@@ -118,6 +52,7 @@
margin-left: 2em; margin-left: 2em;
} </style> } </style>
<link rel="stylesheet" href="slides_files/libs/revealjs/dist/theme/quarto-2740581606dc327e82f18f789b69d8a0.css"> <link rel="stylesheet" href="slides_files/libs/revealjs/dist/theme/quarto-2740581606dc327e82f18f789b69d8a0.css">
<link rel="stylesheet" href="styles.css">
<link href="slides_files/libs/revealjs/plugin/quarto-line-highlight/line-highlight.css" rel="stylesheet"> <link href="slides_files/libs/revealjs/plugin/quarto-line-highlight/line-highlight.css" rel="stylesheet">
<link href="slides_files/libs/revealjs/plugin/reveal-menu/menu.css" rel="stylesheet"> <link href="slides_files/libs/revealjs/plugin/reveal-menu/menu.css" rel="stylesheet">
<link href="slides_files/libs/revealjs/plugin/reveal-menu/quarto-menu.css" rel="stylesheet"> <link href="slides_files/libs/revealjs/plugin/reveal-menu/quarto-menu.css" rel="stylesheet">
...@@ -215,33 +150,71 @@ ...@@ -215,33 +150,71 @@
<div class="slides"> <div class="slides">
<section id="title-slide" class="quarto-title-block center"> <section id="title-slide" class="quarto-title-block center">
<h1 class="title">Mechanistic interpretability for enhancing RAG models</h1>
<div class="quarto-title-authors"> <div style="justify-content: space-between">
<div class="quarto-title-author"> <img src="images/LIFO.png">
<div class="quarto-title-author-name"> <img src="images/Logo Minerve_RVB.jpg" style="width: 375px">
Marine Delvallez <img src="images/France_2030_Logo_rouge_bleu_transparent.png" style="width: 200px">
</div> </div>
</div> <p class="title_perso">Mechanistic interpretability for enhancing RAG models</p>
</div> <p class="subtitle_perso">Internship - Master 2 ARIAS Minerve</p>
<p class="author_perso">Marine Delvallez</p>
</section><section id="TOC"> </section><section id="TOC">
<nav role="doc-toc"> <nav role="doc-toc">
<h2 id="toc-title">Table of contents</h2> <h2 id="toc-title">Table of contents</h2>
<ul> <ul>
<li><a href="#/introduction-context" id="/toc-introduction-context">Introduction &amp; Context</a>
<ul>
<li><a href="#/retrieval-augmented-generation---rag" id="/toc-retrieval-augmented-generation---rag">Retrieval Augmented Generation - RAG</a></li> <li><a href="#/retrieval-augmented-generation---rag" id="/toc-retrieval-augmented-generation---rag">Retrieval Augmented Generation - RAG</a></li>
<li><a href="#/explainability-in-artificial-intelligence" id="/toc-explainability-in-artificial-intelligence">Explainability in Artificial Intelligence</a></li> <li><a href="#/explainability-in-artificial-intelligence" id="/toc-explainability-in-artificial-intelligence">Explainability in Artificial Intelligence</a></li>
<li><a href="#/presentation-and-demonstration-of-mechir" id="/toc-presentation-and-demonstration-of-mechir">Presentation and Demonstration of MechIR</a></li> <li><a href="#/mechanistic-interpretability" id="/toc-mechanistic-interpretability">Mechanistic Interpretability</a></li>
<li><a href="#/use-case-of-my-internship" id="/toc-use-case-of-my-internship">Use case of my internship</a></li>
</ul></li>
<li><a href="#/method" id="/toc-method">Method</a></li>
<li><a href="#/experimentation" id="/toc-experimentation">Experimentation</a></li>
<li><a href="#/conclusion-perspectives" id="/toc-conclusion-perspectives">Conclusion &amp; Perspectives</a>
<ul>
</ul></li>
</ul> </ul>
</nav> </nav>
</section> </section>
<section> <section>
<section id="introduction-context" class="title-slide slide level1 center">
<h1>Introduction &amp; Context</h1>
</section>
<section id="retrieval-augmented-generation---rag" class="title-slide slide level2 center"> <section id="retrieval-augmented-generation---rag" class="title-slide slide level2 center">
<h2>Retrieval Augmented Generation - RAG</h2> <h2>Retrieval Augmented Generation - RAG</h2>
</section> </section>
<section id="retrieval-augmented-generation---definition" class="slide level3"> <section id="retrieval-augmented-generation---definition" class="slide level3">
<h3>Retrieval Augmented Generation - Definition</h3> <h3>Retrieval Augmented Generation - Definition</h3>
<p><em>Add a knowledge database to a generative model</em></p>
<ul>
<li>increase knowledge reliability</li>
<li>release the burden of knowledge on the generator</li>
</ul>
<div class="fragment">
<p>Tasks:</p>
<ul>
<li><em>Knowledge intensive Generation tasks</em></li>
<li>Ex: Question Answering, Fact Checking, Dialogue, Retrieval</li>
</ul>
</div>
</section>
<section id="retrieval" class="slide level3">
<h3>Retrieval</h3>
<p>Models includes Information Retrieval as a sub-task : Identify documents and informations useful to help generation</p>
<p>2 principal technologies:</p>
<ul>
<li>cross-encoder</li>
<li>bi-encoder</li>
</ul>
<p><span class="math inline">\(\Rightarrow\)</span> Needs indexing before use</p>
</section>
<section id="rag-architectures" class="slide level3">
<h3>RAG Architectures</h3>
<div class="quarto-figure quarto-figure-center"> <div class="quarto-figure quarto-figure-center">
<figure> <figure>
<p><img data-src="images/DefRAG.drawio.png"></p> <p><img data-src="images/DefRAG.drawio.png"></p>
...@@ -252,16 +225,27 @@ Marine Delvallez ...@@ -252,16 +225,27 @@ Marine Delvallez
<div class="quarto-figure quarto-figure-center"> <div class="quarto-figure quarto-figure-center">
<figure> <figure>
<p><img data-src="images/DefRAGAvance_integration.drawio.png"></p> <p><img data-src="images/DefRAGAvance_integration.drawio.png"></p>
<figcaption>Advanced RAG Achitecture <span class="citation" data-cites="fan_survey_2024">(<a href="#/references" role="doc-biblioref" onclick="">Fan et al. 2024</a>)</span></figcaption> <figcaption>Advanced RAG Architecture <span class="citation" data-cites="fan_survey_2024">(<a href="#/references" role="doc-biblioref" onclick="">Fan et al. 2024</a>)</span></figcaption>
</figure>
</div>
</div>
</section>
<section id="examples" class="slide level3">
<h3>Examples</h3>
<div class="quarto-figure quarto-figure-center">
<figure>
<p><img data-src="images/RAGHN.drawio.png"></p>
<figcaption>RAG Architecture from <span class="citation" data-cites="tran_retrieval_2024">(<a href="#/references" role="doc-biblioref" onclick="">Tran et al. 2024</a>)</span></figcaption>
</figure> </figure>
</div> </div>
<div class="quarto-figure quarto-figure-center">
<figure>
<p><img data-src="images/RAG4Def.drawio.png"></p>
<figcaption>RAG architecture used in the following</figcaption>
</figure>
</div> </div>
</section> </section>
<section id="example" class="slide level3 center">
<h3>Example</h3>
<img data-src="images/RAGHN-perso.drawio.png" class="r-stretch quarto-figure-center"><p class="caption">RAG Architecture used here (inspired from <span class="citation" data-cites="tran_retrieval_2024">(<a href="#/references" role="doc-biblioref" onclick="">Tran et al. 2024</a>)</span>)</p></section></section>
<section>
<section id="explainability-in-artificial-intelligence" class="title-slide slide level2 center"> <section id="explainability-in-artificial-intelligence" class="title-slide slide level2 center">
<h2>Explainability in Artificial Intelligence</h2> <h2>Explainability in Artificial Intelligence</h2>
...@@ -293,22 +277,23 @@ Marine Delvallez ...@@ -293,22 +277,23 @@ Marine Delvallez
<section id="explanation-through-creation-of-a-model" class="slide level3"> <section id="explanation-through-creation-of-a-model" class="slide level3">
<h3>Explanation through creation of a model</h3> <h3>Explanation through creation of a model</h3>
<img data-src="images/VieModele-TempsXAI.drawio.png" class="r-stretch quarto-figure-center"><p class="caption">Explanation through creation of a model</p></section></section> <img data-src="images/IllVieModele-TempsXAI.drawio.png" class="r-stretch"></section>
<section>
<section id="presentation-and-demonstration-of-mechir" class="title-slide slide level2 center"> <section id="mechanistic-interpretability" class="title-slide slide level2 center">
<h2>Presentation and Demonstration of MechIR</h2> <h2>Mechanistic Interpretability</h2>
</section> </section>
<section id="mechir" class="slide level3"> <section id="definition" class="slide level3">
<h3>MechIR</h3> <h3>Definition</h3>
<h4 id="mechanistic-interpretability">Mechanistic interpretability</h4> <blockquote>
<p>Understand the internal mechanisms of neural networks by <strong>performing causal interventions</strong> on specific model components</p> <p>Reverse-engineer neural networks by identifying internal components (neurons, circuits, activation pathways, …) that either contribute to or causally influence specific behaviors. <span class="citation" data-cites="somvanshi_bridging_2026">(<a href="#/references" role="doc-biblioref" onclick="">Somvanshi et al. 2026</a>)</span></p>
<h4 id="mechir-parry_mechir_2025">MechIR <span class="citation" data-cites="parry_mechir_2025">(<a href="#/references" role="doc-biblioref" onclick="">Parry et al. 2025</a>)</span></h4> </blockquote>
<p>Several approches:</p>
<ul> <ul>
<li><p>Encoder-only models</p></li> <li>Manual Circuit Tracing</li>
<li><p>For Information Retrieval models</p></li> <li><em>Intervention-based Techniques</em></li>
<li><p>Identify components responsible for some behavior</p></li> <li>Representation Analysis</li>
<li><p>Activation Patching Technique</p></li> <li>Toy Models and Synthetic Tasks</li>
</ul> </ul>
</section> </section>
<section id="activation-patching-chen_axiomatic_2024" class="slide level3 smaller"> <section id="activation-patching-chen_axiomatic_2024" class="slide level3 smaller">
...@@ -348,78 +333,84 @@ Let <span class="math inline">\(Q \times \tilde{D}\)</span> the same set of pair ...@@ -348,78 +333,84 @@ Let <span class="math inline">\(Q \times \tilde{D}\)</span> the same set of pair
<li class="fragment"><span class="math inline">\(P = \frac{\bar{p} - p_\hat{D} }{p_\check{D} - p_\hat{D}}\)</span> gives the impact of the perturbation on the model performance</li> <li class="fragment"><span class="math inline">\(P = \frac{\bar{p} - p_\hat{D} }{p_\check{D} - p_\hat{D}}\)</span> gives the impact of the perturbation on the model performance</li>
</ol> </ol>
</section> </section>
<section id="animation-de-lexecution-de-activation-patching" class="slide level3"> <section id="animation-de-lexecution-de-activation-patching-step-0" class="slide level3">
<h3>Animation de l’execution de Activation patching</h3> <h3>Animation de l’execution de Activation patching (Step 0)</h3>
</section>
<section id="step-1-choose-a-perturbation" class="slide level3"> <img data-src="images/activationpatching/Step0_ActivationPatching.drawio.png" class="r-stretch"></section>
<h3>Step 1: Choose a perturbation</h3> <section id="animation-de-lexecution-de-activation-patching-step-1" class="slide level3">
<p>Function that applies the same modification on each document.<br> <h3>Animation de l’execution de Activation patching (Step 1)</h3>
Example :</p>
<div id="107f0802" class="cell" data-execution_count="2"> <img data-src="images/activationpatching/Step1_ActivationPatching.drawio.png" class="r-stretch"></section>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb1"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href=""></a><span class="at">@perturbation</span></span> <section id="animation-de-lexecution-de-activation-patching-step-2" class="slide level3">
<span id="cb1-2"><a href=""></a><span class="kw">def</span> pert1(doc:<span class="bu">str</span>) <span class="op">-&gt;</span> <span class="bu">str</span> :</span> <h3>Animation de l’execution de Activation patching (Step 2)</h3>
<span id="cb1-3"><a href=""></a> <span class="cf">return</span> doc.replace(<span class="st">"solution"</span>, <span class="st">"answer"</span>)</span>
<span id="cb1-4"><a href=""></a></span> <img data-src="images/activationpatching/Step2_ActivationPatching.drawio.png" class="r-stretch"></section>
<span id="cb1-5"><a href=""></a><span class="at">@perturbation</span></span> <section id="animation-de-lexecution-de-activation-patching-step-3-and-4" class="slide level3">
<span id="cb1-6"><a href=""></a><span class="kw">def</span> pert2(doc:<span class="bu">str</span>) <span class="op">-&gt;</span> <span class="bu">str</span>:</span> <h3>Animation de l’execution de Activation patching (Step 3 and 4)</h3>
<span id="cb1-7"><a href=""></a> <span class="cf">return</span> doc.replace(<span class="st">"microwave"</span>, <span class="st">"toaster"</span>)</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
</div> <img data-src="images/activationpatching/Step3point_ActivationPatching.drawio.png" class="r-stretch"></section>
<section id="animation-de-lexecution-de-activation-patching-step-3-and-4-1" class="slide level3">
<h3>Animation de l’execution de Activation patching (Step 3 and 4)</h3>
<img data-src="images/activationpatching/Step3_ActivationPatching.drawio.png" class="r-stretch"></section>
<section id="mechir-parry_mechir_2025" class="slide level3">
<h3>MechIR <span class="citation" data-cites="parry_mechir_2025">(<a href="#/references" role="doc-biblioref" onclick="">Parry et al. 2025</a>)</span></h3>
<ul>
<li>Python library</li>
<li>Activation Patching</li>
<li>Information retrieval models</li>
<li>Charts sensibility of model components to a perturbation</li>
</ul>
</section> </section>
<section id="what-is-a-good-perturbation" class="slide level3">
<h3>What is a good perturbation</h3> <section id="use-case-of-my-internship" class="title-slide slide level2 small center">
<h2>Use case of my internship</h2>
<p><em>Use case</em> : Domain specific language</p>
<p>Considering that activation patching charts the sensibility of a model to a perturbation</p>
<p>Hypothesis :</p>
<ul> <ul>
<li>Has an impact on the representation of documents</li> <li>Models have <strong>specific components dedicated to domain specific language</strong></li>
<li>Be useful for interpretation</li> <li>This domain specific language dedicated part of the model can be <strong>charted with activation patching</strong></li>
<li>That map can be used to <strong>enhance model comprehension and robustness</strong> of the domain specific language dedicated part of the model</li>
</ul> </ul>
</section>
</section>
<section>
<section id="method" class="title-slide slide level1 center">
<h1>Method</h1>
<img data-src="images/perturbation-score.png" class="r-stretch quarto-figure-center"><p class="caption">Perturbation Score</p></section> </section>
<section id="perturbation-creation-technique" class="slide level3"> <section id="building-a-perturbation" class="slide level3">
<h3>Perturbation creation technique</h3> <h3>Building a perturbation</h3>
<ol type="1"> <ol type="1">
<li>Identify vocabulary specific to the dataset</li> <li>Identify domain specific language <span class="math inline">\(L\)</span> of the dataset
<li>Find in the vocabulary words with several meaning <span class="math inline">\(m_D\)</span> and <span class="math inline">\(m_D\)</span></li> <ul>
<li>Replace that word by a synonym of the <span class="math inline">\(m_D\)</span> meaning</li> <li>List words from each text</li>
<li>Calculate frequency and TF-IDF of each word</li>
<li>Find the domain specific language</li>
</ul></li>
<li>Identify polysemic words <span class="math inline">\(w\)</span> that has a different meaning in everyday language</li>
<li>Find a synonym <span class="math inline">\(s\)</span> of the everyday meaning of each <span class="math inline">\(w\)</span></li>
<li>Build perturbation <span class="math inline">\(w \to s\)</span></li>
</ol> </ol>
</section> </section>
<section id="step-2-instantiate-the-model-and-load-data" class="slide level3"> <section id="examples-of-perturbation" class="slide level3">
<h3>Step 2 : Instantiate the model and load data</h3> <h3>Examples of perturbation</h3>
<div id="1e6eed38" class="cell" data-execution_count="3"> <ul>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb2"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href=""></a>dot_model_name <span class="op">=</span> <span class="st">"sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco"</span></span> <li>solution -&gt; answer
<span id="cb2-2"><a href=""></a>dot_model <span class="op">=</span> Dot(dot_model_name)</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div> <ul>
<div class="cell-output cell-output-stdout"> <li><em>microwave a measurements of dielectric absorption in dilute <strong>solutions</strong></em></li>
<pre><code>Moving model to device: cpu <li><em>microwave a measurements of dielectric absorption in dilute answer</em></li>
Loaded pretrained model sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco into HookedEncoder</code></pre> </ul></li>
</div> <li>microwave -&gt; toaster
</div> <ul>
<div id="b6407c32" class="cell" data-execution_count="4"> <li><em><strong>microwave</strong> a measurements of dielectric absorption in dilute solutions</em></li>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb4"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href=""></a>dataset <span class="op">=</span> MechIRDataset(<span class="st">"vaswani"</span>, query_id_subset<span class="op">=</span>[<span class="st">"1"</span>])</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div> <li><em><strong>toaster</strong> a measurements of dielectric absorption in dilute solutions</em></li>
</div> </ul></li>
<div id="c44e6315" class="cell" data-execution_count="5"> </ul>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb5"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href=""></a>pert1_dot_collator <span class="op">=</span> DotDataCollator(dot_model.tokenizer, pert1, q_max_length<span class="op">=</span><span class="va">None</span>, d_max_length<span class="op">=</span><span class="va">None</span>, perturb_type<span class="op">=</span><span class="st">"replace"</span>)</span> <div id="289c30e5" class="cell" data-execution_count="6">
<span id="cb5-2"><a href=""></a>pert1_dot_dataloader <span class="op">=</span> DataLoader(dataset, batch_size<span class="op">=</span><span class="dv">16</span>, collate_fn<span class="op">=</span>pert1_dot_collator)</span>
<span id="cb5-3"><a href=""></a></span>
<span id="cb5-4"><a href=""></a>pert2_dot_collator <span class="op">=</span> DotDataCollator(dot_model.tokenizer, pert2, q_max_length<span class="op">=</span><span class="va">None</span>, d_max_length<span class="op">=</span><span class="va">None</span>, perturb_type<span class="op">=</span><span class="st">"replace"</span>)</span>
<span id="cb5-5"><a href=""></a>pert2_dot_dataloader <span class="op">=</span> DataLoader(dataset, batch_size<span class="op">=</span><span class="dv">16</span>, collate_fn<span class="op">=</span>pert2_dot_collator)</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
</div>
</section>
<section id="visualisation-of-perturbed-data" class="slide level3">
<h3>Visualisation of perturbed data</h3>
<div id="1bfa5d77" class="cell" data-execution_count="6">
<div class="cell-output cell-output-stdout">
<pre><code>solution -&gt; answer
Query: [CLS] measurement of dielectric constant of liquids by the use of microwave techniques [SEP]
Baseline Document: [CLS] broadband millimetre wave paramagnetic resonance spectrometer the specimen and waveguide which can be cooled by means of a cryostat are placed between close pole pieces giving high uniform magnetic fields design details and some measurements on zero field splittings are given [SEP]
Perturbed Document: [CLS] broadband millimetre wave paramagnetic resonance spectrometer the specimen and waveguide which can be cooled by means of a cryostat are placed between close pole pieces giving high uniform magnetic fields design details and some measurements on zero field splittings are given [SEP]
==================================================
Query: [CLS] measurement of dielectric constant of liquids by the use of microwave techniques [SEP]
Baseline Document: [CLS] microwave measurements of dielectric absorption in dilute solutions [SEP]
Perturbed Document: [CLS] microwave measurements of dielectric absorption in dilute answers [SEP]
==================================================</code></pre>
</div>
</div>
<div id="4b3033e8" class="cell" data-execution_count="7">
<div class="cell-output cell-output-stdout"> <div class="cell-output cell-output-stdout">
<pre><code>microwave -&gt; toaster <pre><code>REPLACE
Query: [CLS] measurement of dielectric constant of liquids by the use of microwave techniques [SEP] Query: [CLS] measurement of dielectric constant of liquids by the use of microwave techniques [SEP]
Baseline Document: [CLS] broadband millimetre wave paramagnetic resonance spectrometer the specimen and waveguide which can be cooled by means of a cryostat are placed between close pole pieces giving high uniform magnetic fields design details and some measurements on zero field splittings are given [SEP] Baseline Document: [CLS] broadband millimetre wave paramagnetic resonance spectrometer the specimen and waveguide which can be cooled by means of a cryostat are placed between close pole pieces giving high uniform magnetic fields design details and some measurements on zero field splittings are given [SEP]
Perturbed Document: [CLS] broadband millimetre wave paramagnetic resonance spectrometer the specimen and waveguide which can be cooled by means of a cryostat are placed between close pole pieces giving high uniform magnetic fields design details and some measurements on zero field splittings are given [SEP] Perturbed Document: [CLS] broadband millimetre wave paramagnetic resonance spectrometer the specimen and waveguide which can be cooled by means of a cryostat are placed between close pole pieces giving high uniform magnetic fields design details and some measurements on zero field splittings are given [SEP]
...@@ -431,43 +422,197 @@ Perturbed Document: [CLS] toaster measurements of dielectric absorption in dilut ...@@ -431,43 +422,197 @@ Perturbed Document: [CLS] toaster measurements of dielectric absorption in dilut
</div> </div>
</div> </div>
</section> </section>
<section id="step-4-measure-the-impact-of-the-perturbation-on-the-model" class="slide level3"> <section id="select-useful-perturbation" class="slide level3">
<h3>Step 4 : Measure the impact of the perturbation on the model</h3> <h3>Select useful Perturbation</h3>
<div id="d6d33dae" class="cell" data-execution_count="8"> <ol start="5" type="1">
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb8"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href=""></a><span class="co"># Initialize lists to store baseline and perturbed performances for each dataloader</span></span> <li>Apply Activation Patching</li>
<span id="cb8-2"><a href=""></a>all_baseline_performance <span class="op">=</span> {<span class="st">"pert1"</span>: [], <span class="st">"pert2"</span>: []}</span> <li>Select relevant perturbations</li>
<span id="cb8-3"><a href=""></a>all_perturbed_performance <span class="op">=</span> {<span class="st">"pert1"</span>: [], <span class="st">"pert2"</span>: []}</span> </ol>
<span id="cb8-4"><a href=""></a></span> <div id="1bb1c95a" class="cell" data-execution_count="7">
<span id="cb8-5"><a href=""></a><span class="co"># Calculate performances for each perturbation_type</span></span>
<span id="cb8-6"><a href=""></a>calculate_performance(dot_model, pert1_dot_dataloader, all_baseline_performance[<span class="st">"pert1"</span>], all_perturbed_performance[<span class="st">"pert1"</span>])</span>
<span id="cb8-7"><a href=""></a>calculate_performance(dot_model, pert2_dot_dataloader, all_baseline_performance[<span class="st">"pert2"</span>], all_perturbed_performance[<span class="st">"pert2"</span>])</span>
<span id="cb8-8"><a href=""></a></span>
<span id="cb8-9"><a href=""></a>plot_score_dists_mult(all_baseline_performance, all_perturbed_performance, plot_type<span class="op">=</span><span class="st">"kde"</span>)</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
<div class="cell-output cell-output-display"> <div class="cell-output cell-output-display">
<div> <div>
<figure> <figure>
<p><img data-src="slides_files/figure-revealjs/cell-9-output-1.png" width="1430" height="381"></p> <p><img data-src="slides_files/figure-revealjs/cell-7-output-1.png" width="1430" height="381"></p>
</figure> </figure>
</div> </div>
</div> </div>
</div> </div>
</section> </section>
<section id="step-5-chart-the-sensitivity-of-the-model-to-the-perturbation" class="slide level3"> <section id="chart-model-sensibility-to-the-perturbation" class="slide level3">
<h3>Step 5 : Chart the sensitivity of the model to the perturbation</h3> <h3>Chart Model sensibility to the perturbation</h3>
<div id="e08b7f31" class="cell" data-execution_count="9"> <ol start="7" type="1">
<div class="cell-output cell-output-display"> <li>Map the model sensibility to the perturbation</li>
<div> </ol>
<figure>
<p><img data-src="slides_files/figure-revealjs/cell-10-output-1.png" width="765" height="523"></p> <img data-src="images/chart_append-microwave_TASB_vaswani.png" class="r-stretch"></section>
</figure> <section id="finetune-the-model" class="slide level3">
</div> <h3>Finetune the Model</h3>
</div> <ol start="8" type="1">
</div> <li>Modify sensible weights</li>
<!-- ![Chart of the impact of a perturbation on the components](images/ActivationPatchingAllHeadExempleMechIR.png) --> </ol>
<p>WORK IN PROGRESS</p>
</section></section>
<section>
<section id="experimentation" class="title-slide slide level1 center">
<h1>Experimentation</h1>
</section>
<section id="setup---dataset" class="slide level3">
<h3>Setup - Dataset</h3>
<div class="columns">
<div class="column" style="width:50%;">
<ul>
<li>Extracted from EvalLLM2026 Challenge dataset</li>
<li>French Defense dataset</li>
<li>Pairs <code>(question, chunk of document)</code></li>
<li>52 pairs, 26 questions</li>
</ul>
</div><div class="column" style="width:50%;">
<p><img data-src="images/ChallengeEvalLLM-pagewebPetit.png"></p>
</div></div>
</section>
<section id="setup---model" class="slide level3">
<h3>Setup - Model</h3>
<p>E5-small <span class="citation" data-cites="wang_text_2022">(<a href="#/references" role="doc-biblioref" onclick="">Wang et al. 2022</a>)</span></p>
<ul>
<li>Constrative bi-encoder</li>
<li>Trained on unlabeled pairs (query, passage) CCPairs</li>
<li>Based on MiniLM <span class="citation" data-cites="wang_minilmv2_2021">(<a href="#/references" role="doc-biblioref" onclick="">Wang et al. 2021</a>)</span></li>
<li>33M params</li>
</ul>
</section>
<section id="setup---mechanistic-interpretability-tools" class="slide level3">
<h3>Setup - Mechanistic Interpretability Tools</h3>
<p>Python libraries:</p>
<ul>
<li>MechIR <span class="citation" data-cites="parry_mechir_2025">(<a href="#/references" role="doc-biblioref" onclick="">Parry et al. 2025</a>)</span></li>
<li>TransformerLens <span class="citation" data-cites="nanda_transformerlens_2022">(<a href="#/references" role="doc-biblioref" onclick="">Nanda and Bloom 2022</a>)</span></li>
</ul>
</section> </section>
<section id="perspective-enhance-a-model-with-mechir" class="slide level3"> <section id="first-results---domain-specific-language-identification" class="slide level3">
<h3>Perspective : Enhance a model with MechIR</h3> <h3>First Results - Domain specific language identification</h3>
<p>TODO</p> <ul>
<li>Defense-specific language is prevalent</li>
</ul>
<table class="caption-top">
<thead>
<tr class="header">
<th>Word</th>
<th>Frequency</th>
<th>TF-IDF score</th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td>militaire</td>
<td>26</td>
<td>0.04494102684257385</td>
</tr>
<tr class="even">
<td>drone</td>
<td>21</td>
<td>0.04081921309724857</td>
</tr>
<tr class="odd">
<td>article</td>
<td>13</td>
<td>0.0317659447734224</td>
</tr>
<tr class="even">
<td>défense</td>
<td>9</td>
<td>0.018444049962599192</td>
</tr>
<tr class="odd">
<td>emploi</td>
<td>7</td>
<td>0.017992160135997803</td>
</tr>
<tr class="even">
<td>armée</td>
<td>7</td>
<td>0.0166576415296721</td>
</tr>
<tr class="odd">
<td>…</td>
<td>…</td>
<td>…</td>
</tr>
</tbody>
</table>
</section>
<section id="first-results---perturbations-in-general" class="slide level3">
<h3>First results - Perturbations in general</h3>
<div class="columns">
<div class="column" style="width:50%;">
<p><img data-src="images/a%C3%A9rien%20-%20l%C3%A9ger_scores_paires-mainV1_exposé.png"> <img data-src="images/d%C3%A9fense%20-%20corne_scores_paires-mainV1_exposé.png"></p>
</div><div class="column" style="width:50%;">
<p><img data-src="images/article%20-%20produit_scores_paires-mainV1_exposé.png"> <img data-src="images/protection%20-%20bouclier_scores_paires-mainV1_exposé.png"></p>
</div></div>
</section>
<section id="first-results---best-perturbations" class="slide level3 smaller">
<h3>First results - Best Perturbations</h3>
<div class="columns">
<div class="column" style="width:50%;">
<p><img data-src="images/emploi%20-%20poste_scores_paires-mainV1_exposé.png"></p>
<p><img data-src="images/militaire%20-%20soldat_scores_paires-mainV1_expos%C3%A9.png"></p>
</div><div class="column" style="width:50%;">
<p><img data-src="images/syst%C3%A8me%20-%20ensemble%20d'%C3%A9quation_scores_paires-mainV1_expos%C3%A9.png"></p>
<ul>
<li>needs more complex perturbation (grammar, group of words)</li>
<li>not enough</li>
</ul>
<p>. . .</p>
<p>LLMs target polysemy problem of models like Word2Vect</p>
</div></div>
</section>
<section id="work-left-for-the-next-25-months" class="slide level3">
<h3>Work left for the next 2,5 months</h3>
<ul>
<li class="fragment">Identify new perturbations
<ul>
<li class="fragment">new words and synonyms</li>
<li class="fragment">new type of perturbation</li>
</ul></li>
<li class="fragment">Chart the sensibility of the model to new perturbations</li>
<li class="fragment">Identify model modifications that enhance domain specific language comprehension/ use</li>
</ul>
</section></section>
<section>
<section id="conclusion-perspectives" class="title-slide slide level1 center">
<h1>Conclusion &amp; Perspectives</h1>
</section>
<section id="conclusion" class="slide level3">
<h3>Conclusion</h3>
<ul>
<li>H1 : Models have specific components dedicated to domain specific language</li>
<li>H2 : This domain specific language dedicated part of the model can be charted with activation patching</li>
<li>H3 : That map can be used to enhance model comprehension and robustness of the domain specific language dedicated part of the model</li>
</ul>
<p><span class="math inline">\(\Rightarrow\)</span> Work in Progress</p>
</section>
<section id="other-perturbations-and-linguistic-structures" class="slide level3">
<h3>Other perturbations and linguistic structures</h3>
<ul>
<li>subgroups of domain specific language</li>
<li>temporal relationships</li>
<li>negation</li>
<li>expression of diachrony</li>
</ul>
</section>
<section id="other-use-of-the-map" class="slide level3">
<h3>Other use of the map</h3>
<ul>
<li>Transfert learning</li>
<li>Prunning</li>
<li>Specialization of general models</li>
</ul>
</section>
<section id="section" class="title-slide slide level2 remerciements center">
<h2></h2>
<p>Thanks for you attention</p>
<p>Questions</p>
</section> </section>
<section id="references" class="slide level3 smaller scrollable"> <section id="references" class="slide level3 smaller scrollable">
<h3>References</h3> <h3>References</h3>
...@@ -484,15 +629,28 @@ Fan, Wenqi, Yujuan Ding, Liangbo Ning, et al. 2024. <em>A <span>Survey</span> on ...@@ -484,15 +629,28 @@ Fan, Wenqi, Yujuan Ding, Liangbo Ning, et al. 2024. <em>A <span>Survey</span> on
<div id="ref-lewis_retrieval-augmented_2020" class="csl-entry" role="listitem"> <div id="ref-lewis_retrieval-augmented_2020" class="csl-entry" role="listitem">
Lewis, Patrick, Ethan Perez, Aleksandra Piktus, et al. 2020. <span>“Retrieval-<span>Augmented</span> <span>Generation</span> for <span>Knowledge</span>-<span>Intensive</span> <span>NLP</span> <span>Tasks</span>.”</span> <em>arXiv: Computation and Language</em>. Lewis, Patrick, Ethan Perez, Aleksandra Piktus, et al. 2020. <span>“Retrieval-<span>Augmented</span> <span>Generation</span> for <span>Knowledge</span>-<span>Intensive</span> <span>NLP</span> <span>Tasks</span>.”</span> <em>arXiv: Computation and Language</em>.
</div> </div>
<div id="ref-nanda_transformerlens_2022" class="csl-entry" role="listitem">
Nanda, Neel, and Joseph Bloom. 2022. <em><span>TransformerLens</span></em>. <a href="https://github.com/TransformerLensOrg/TransformerLens">https://github.com/TransformerLensOrg/TransformerLens</a>.
</div>
<div id="ref-parry_mechir_2025" class="csl-entry" role="listitem"> <div id="ref-parry_mechir_2025" class="csl-entry" role="listitem">
Parry, Andrew, Catherine Chen, Carsten Eickhoff, and Sean MacAvaney. 2025. <span>“<span>MechIR</span>: <span>A</span> <span>Mechanistic</span> <span>Interpretability</span> <span>Framework</span> for <span>Information</span> <span>Retrieval</span>.”</span> <em>Advances in <span>Information</span> <span>Retrieval</span> - 47th <span>European</span> <span>Conference</span> on <span>Information</span> <span>Retrieval</span>, <span>ECIR</span> 2025, <span>Lucca</span>, <span>Italy</span>, <span>April</span> 6-10, 2025, <span>Proceedings</span>, <span>Part</span> <span>V</span></em>, Lecture <span>Notes</span> in <span>Computer</span> <span>Science</span>, vol. 15576: 89–95. <a href="https://doi.org/10.1007/978-3-031-88720-8_16">https://doi.org/10.1007/978-3-031-88720-8_16</a>. Parry, Andrew, Catherine Chen, Carsten Eickhoff, and Sean MacAvaney. 2025. <span>“<span>MechIR</span>: <span>A</span> <span>Mechanistic</span> <span>Interpretability</span> <span>Framework</span> for <span>Information</span> <span>Retrieval</span>.”</span> <em>Advances in <span>Information</span> <span>Retrieval</span> - 47th <span>European</span> <span>Conference</span> on <span>Information</span> <span>Retrieval</span>, <span>ECIR</span> 2025, <span>Lucca</span>, <span>Italy</span>, <span>April</span> 6-10, 2025, <span>Proceedings</span>, <span>Part</span> <span>V</span></em>, Lecture <span>Notes</span> in <span>Computer</span> <span>Science</span>, vol. 15576: 89–95. <a href="https://doi.org/10.1007/978-3-031-88720-8_16">https://doi.org/10.1007/978-3-031-88720-8_16</a>.
</div> </div>
<div id="ref-somvanshi_bridging_2026" class="csl-entry" role="listitem">
Somvanshi, Shriyank, Md Monzurul Islam, Amir Rafe, et al. 2026. <span>“Bridging the <span>Black</span> <span>Box</span>: <span>A</span> <span>Survey</span> on <span>Mechanistic</span> <span>Interpretability</span> in <span>AI</span>.”</span> <em>ACM Computing Surveys</em> 58 (8): 1–35. <a href="https://doi.org/10.1145/3787104">https://doi.org/10.1145/3787104</a>.
</div>
<div id="ref-tran_retrieval_2024" class="csl-entry" role="listitem"> <div id="ref-tran_retrieval_2024" class="csl-entry" role="listitem">
Tran, The Trung, Carlos-Emiliano González-Gallardo, and Antoine Doucet. 2024. <span>“Retrieval <span>Augmented</span> <span>Generation</span> for <span>Historical</span> <span>Newspapers</span>.”</span> <em>Proceedings of the 24th <span>ACM</span>/<span>IEEE</span> <span>Joint</span> <span>Conference</span> on <span>Digital</span> <span>Libraries</span></em> (Hong Kong China), December, 1–5. <a href="https://doi.org/10.1145/3677389.3702542">https://doi.org/10.1145/3677389.3702542</a>. Tran, The Trung, Carlos-Emiliano González-Gallardo, and Antoine Doucet. 2024. <span>“Retrieval <span>Augmented</span> <span>Generation</span> for <span>Historical</span> <span>Newspapers</span>.”</span> <em>Proceedings of the 24th <span>ACM</span>/<span>IEEE</span> <span>Joint</span> <span>Conference</span> on <span>Digital</span> <span>Libraries</span></em> (Hong Kong China), December, 1–5. <a href="https://doi.org/10.1145/3677389.3702542">https://doi.org/10.1145/3677389.3702542</a>.
</div> </div>
<div id="ref-wang_text_2022" class="csl-entry" role="listitem">
Wang, Liang, Nan Yang, Xiaolong Huang, et al. 2022. <em>Text <span>Embeddings</span> by <span>Weakly</span>-<span>Supervised</span> <span>Contrastive</span> <span>Pre</span>-Training</em>. <a href="https://arxiv.org/abs/2212.03533v2">https://arxiv.org/abs/2212.03533v2</a>.
</div>
<div id="ref-wang_minilmv2_2021" class="csl-entry" role="listitem">
Wang, Wenhui, Hangbo Bao, Shaohan Huang, Li Dong, and Furu Wei. 2021. <span>“<span>MiniLMv2</span>: <span>Multi</span>-<span>Head</span> <span>Self</span>-<span>Attention</span> <span>Relation</span> <span>Distillation</span> for <span>Compressing</span> <span>Pretrained</span> <span>Transformers</span>.”</span> In <em>Findings of the <span>Association</span> for <span>Computational</span> <span>Linguistics</span>: <span>ACL</span>-<span>IJCNLP</span> 2021</em>, edited by Chengqing Zong, Fei Xia, Wenjie Li, and Roberto Navigli. Association for Computational Linguistics. <a href="https://doi.org/10.18653/v1/2021.findings-acl.188">https://doi.org/10.18653/v1/2021.findings-acl.188</a>.
</div>
</div> </div>
</section></section> </section>
</section>
</div> </div>
<div class="quarto-auto-generated-content" style="display: none;"> <div class="quarto-auto-generated-content" style="display: none;">
<div class="footer footer-default"> <div class="footer footer-default">
...@@ -549,7 +707,7 @@ Tran, The Trung, Carlos-Emiliano González-Gallardo, and Antoine Doucet. 2024. < ...@@ -549,7 +707,7 @@ Tran, The Trung, Carlos-Emiliano González-Gallardo, and Antoine Doucet. 2024. <
progress: true, progress: true,
// Display the page number of the current slide // Display the page number of the current slide
slideNumber: false, slideNumber: 'c/t',
// 'all', 'print', or 'speaker' // 'all', 'print', or 'speaker'
showSlideNumber: 'all', showSlideNumber: 'all',
......
seminaireCA/slides.qmd
View file @ 2218e5d6
--- ---
title: "Mechanistic interpretability for enhancing RAG models" title: "Mechanistic interpretability for enhancing RAG models"
subtitle: "Internship - Master 2 ARIAS Minerve"
author: Marine Delvallez author: Marine Delvallez
format: format:
revealjs: revealjs:
...@@ -7,39 +8,65 @@ format: ...@@ -7,39 +8,65 @@ format:
toc-depth: 2 toc-depth: 2
slide-level: 3 slide-level: 3
mouse-wheel: true mouse-wheel: true
template-partials:
- title-slide.html
jupyter: venv-mechir jupyter: venv-mechir
execute: execute:
cache: true cache: true
freeze: auto freeze: auto
bibliography: biblio.bib bibliography: biblio.bib
slide-number: true
style: | style: |
.columns {
display: grid; css: styles.css
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 1rem;
}
.small {
font-size: 20px
}
.midsize{
font-size: 25px
}
--- ---
# Introduction & Context
## Retrieval Augmented Generation - RAG ## Retrieval Augmented Generation - RAG
### Retrieval Augmented Generation - Definition ### Retrieval Augmented Generation - Definition
_Add a knowledge database to a generative model_
- increase knowledge reliability
- release the burden of knowledge on the generator
. . .
Tasks:
- _Knowledge intensive Generation tasks_
- Ex: Question Answering, Fact Checking, Dialogue, Retrieval
### Retrieval
Models includes Information Retrieval as a sub-task : Identify documents and informations useful to help generation
2 principal technologies:
- cross-encoder
- bi-encoder
$\Rightarrow$ Needs indexing before use
### RAG Architectures
![Simple RAG Architecture [@lewis_retrieval-augmented_2020]](images/DefRAG.drawio.png) ![Simple RAG Architecture [@lewis_retrieval-augmented_2020]](images/DefRAG.drawio.png)
. . . . . .
![Advanced RAG Achitecture [@fan_survey_2024]](images/DefRAGAvance_integration.drawio.png) ![Advanced RAG Architecture [@fan_survey_2024]](images/DefRAGAvance_integration.drawio.png)
### Examples
![RAG Architecture from [@tran_retrieval_2024]](images/RAGHN.drawio.png)
### Example {.center} ![RAG architecture used in the following](images/RAG4Def.drawio.png)
![RAG Architecture used here (inspired from [@tran_retrieval_2024])](images/RAGHN-perso.drawio.png)
## Explainability in Artificial Intelligence ## Explainability in Artificial Intelligence
...@@ -76,11 +103,81 @@ _Why ?_ ...@@ -76,11 +103,81 @@ _Why ?_
### Explanation through creation of a model ### Explanation through creation of a model
![Explanation through creation of a model](images/VieModele-TempsXAI.drawio.png)
## Presentation and Demonstration of MechIR ![](images/IllVieModele-TempsXAI.drawio.png)
## Mechanistic Interpretability
### Definition
> Reverse-engineer neural networks by identifying internal components (neurons, circuits, activation pathways, ...) that either contribute to or causally influence specific behaviors. [@somvanshi_bridging_2026]
Several approches:
- Manual Circuit Tracing
- _Intervention-based Techniques_
- Representation Analysis
- Toy Models and Synthetic Tasks
### Activation Patching [@chen_axiomatic_2024] {.smaller}
Let $Q \times D \subset \mathcal{Q}\times\mathcal{D}$ be a set of pairs of questions and documents
Let $Q \times \tilde{D}$ the same set of pairs but with perturbed documents
:::{.incremental}
1. Forward pass all $Q\times D$
- record $o_{i,j}^e$ the output of each component $n_{i,j}, \forall e \in Q\times D$
- record $p_D$ the performance of the model
2. Forward pass all $Q\times \tilde{D}$
- record $o_{i,j}^\tilde{e}$ the output of each component $n_{i,j}, \forall \tilde{e} \in Q\times \tilde{D}$
- record $p_\tilde{D}$ the performance of the model
3. Rewrite $D, e, \tilde{D} \text{ and } \tilde{e}$ as
- $\hat{D}, \hat{e}, \check{D} \text{ and } \check{e}$ if $p_D > p_\tilde{D}$
- $\check{D}, \check{e}, \hat{D} \text{ and } \hat{e}$ otherwise
:::
### Activation Patching [@chen_axiomatic_2024] {.smaller}
3. Rewrite $D, e, \tilde{D} \text{ and } \tilde{e}$ as
- $\hat{D}, \hat{e}, \check{D} \text{ and } \check{e}$ if $p_D > p_\tilde{D}$
- $\check{D}, \check{e}, \hat{D} \text{ and } \hat{e}$ otherwise
:::{.incremental}
4. For each component $n_{i,j}$ forward pass $Q\times\check{D}$ but replace $o_{i,j}^{\check{e}}$ by $o_{i,j}^{\hat{e}}$ for each $\check{e}$. Record the performance $\bar{p}$
5. $P = \frac{\bar{p} - p_\hat{D} }{p_\check{D} - p_\hat{D}}$ gives the impact of the perturbation on the model performance
:::
### Animation de l'execution de Activation patching (Step 0)
![](images/activationpatching/Step0_ActivationPatching.drawio.png)
### Animation de l'execution de Activation patching (Step 1)
![](images/activationpatching/Step1_ActivationPatching.drawio.png)
### MechIR ### Animation de l'execution de Activation patching (Step 2)
![](images/activationpatching/Step2_ActivationPatching.drawio.png)
### Animation de l'execution de Activation patching (Step 3 and 4)
![](images/activationpatching/Step3point_ActivationPatching.drawio.png)
### Animation de l'execution de Activation patching (Step 3 and 4)
![](images/activationpatching/Step3_ActivationPatching.drawio.png)
### MechIR [@parry_mechir_2025]
- Python library
- Activation Patching
- Information retrieval models
- Charts sensibility of model components to a perturbation
```{python} ```{python}
from mechir import Dot from mechir import Dot
...@@ -189,57 +286,51 @@ def plot_score_dists_mult(all_baseline_scores, all_perturbed_scores, plot_type=" ...@@ -189,57 +286,51 @@ def plot_score_dists_mult(all_baseline_scores, all_perturbed_scores, plot_type="
``` ```
#### Mechanistic interpretability
Understand the internal mechanisms of neural networks by **performing causal interventions** on specific model components
#### MechIR [@parry_mechir_2025]
- Encoder-only models
- For Information Retrieval models
- Identify components responsible for some behavior
- Activation Patching Technique
### Activation Patching [@chen_axiomatic_2024] {.smaller}
Let $Q \times D \subset \mathcal{Q}\times\mathcal{D}$ be a set of pairs of questions and documents
Let $Q \times \tilde{D}$ the same set of pairs but with perturbed documents
:::{.incremental} ## Use case of my internship {.small}
1. Forward pass all $Q\times D$ _Use case_ : Domain specific language
- record $o_{i,j}^e$ the output of each component $n_{i,j}, \forall e \in Q\times D$
- record $p_D$ the performance of the model
2. Forward pass all $Q\times \tilde{D}$
- record $o_{i,j}^\tilde{e}$ the output of each component $n_{i,j}, \forall \tilde{e} \in Q\times \tilde{D}$
- record $p_\tilde{D}$ the performance of the model
3. Rewrite $D, e, \tilde{D} \text{ and } \tilde{e}$ as
- $\hat{D}, \hat{e}, \check{D} \text{ and } \check{e}$ if $p_D > p_\tilde{D}$
- $\check{D}, \check{e}, \hat{D} \text{ and } \hat{e}$ otherwise
::: Considering that activation patching charts the sensibility of a model to a perturbation
### Activation Patching [@chen_axiomatic_2024] {.smaller} Hypothesis :
3. Rewrite $D, e, \tilde{D} \text{ and } \tilde{e}$ as - Models have **specific components dedicated to domain specific language**
- $\hat{D}, \hat{e}, \check{D} \text{ and } \check{e}$ if $p_D > p_\tilde{D}$ - This domain specific language dedicated part of the model can be **charted with activation patching**
- $\check{D}, \check{e}, \hat{D} \text{ and } \hat{e}$ otherwise - That map can be used to **enhance model comprehension and robustness** of the domain specific language dedicated part of the model
:::{.incremental} ```{python}
#| output: false
dot_model_name = "sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco"
dot_model = Dot(dot_model_name)
```
4. For each component $n_{i,j}$ forward pass $Q\times\check{D}$ but replace $o_{i,j}^{\check{e}}$ by $o_{i,j}^{\hat{e}}$ for each $\check{e}$. Record the performance $\bar{p}$ ```{python}
5. $P = \frac{\bar{p} - p_\hat{D} }{p_\check{D} - p_\hat{D}}$ gives the impact of the perturbation on the model performance dataset = MechIRDataset("vaswani", query_id_subset=["1"])
```
::: # Method
### Animation de l'execution de Activation patching ### Building a perturbation
1. Identify domain specific language $L$ of the dataset
- List words from each text
- Calculate frequency and TF-IDF of each word
- Find the domain specific language
2. Identify polysemic words $w$ that has a different meaning in everyday language
3. Find a synonym $s$ of the everyday meaning of each $w$
4. Build perturbation $w \to s$
### Examples of perturbation
### Step 1: Choose a perturbation - solution -> answer
- _microwave a measurements of dielectric absorption in dilute **solutions**_
- _microwave a measurements of dielectric absorption in dilute answer_
- microwave -> toaster
- _**microwave** a measurements of dielectric absorption in dilute solutions_
- _**toaster** a measurements of dielectric absorption in dilute solutions_
Function that applies the same modification on each document.
Example :
``` {python} ``` {python}
#| echo: true
@perturbation @perturbation
def pert1(doc:str) -> str : def pert1(doc:str) -> str :
...@@ -251,81 +342,45 @@ def pert2(doc:str) -> str: ...@@ -251,81 +342,45 @@ def pert2(doc:str) -> str:
``` ```
### What is a good perturbation
- Has an impact on the representation of documents
- Be useful for interpretation
![Perturbation Score](images/perturbation-score.png)
### Perturbation creation technique
1. Identify vocabulary specific to the dataset
2. Find in the vocabulary words with several meaning $m_D$ and $m_D$
3. Replace that word by a synonym of the $m_D$ meaning
### Step 2 : Instantiate the model and load data
```{python}
#| echo: true
dot_model_name = "sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco"
dot_model = Dot(dot_model_name)
```
```{python}
#| echo: true
dataset = MechIRDataset("vaswani", query_id_subset=["1"])
```
```{python} ```{python}
#| echo: true #|echo: false
pert1_dot_collator = DotDataCollator(dot_model.tokenizer, pert1, q_max_length=None, d_max_length=None, perturb_type="replace") pert1_dot_collator = DotDataCollator(dot_model.tokenizer, pert1, q_max_length=None, d_max_length=None, perturb_type="replace")
pert1_dot_dataloader = DataLoader(dataset, batch_size=16, collate_fn=pert1_dot_collator) pert1_dot_dataloader = DataLoader(dataset, batch_size=16, collate_fn=pert1_dot_collator)
pert2_dot_collator = DotDataCollator(dot_model.tokenizer, pert2, q_max_length=None, d_max_length=None, perturb_type="replace") pert2_dot_collator = DotDataCollator(dot_model.tokenizer, pert2, q_max_length=None, d_max_length=None, perturb_type="replace")
pert2_dot_dataloader = DataLoader(dataset, batch_size=16, collate_fn=pert2_dot_collator) pert2_dot_dataloader = DataLoader(dataset, batch_size=16, collate_fn=pert2_dot_collator)
``` print("REPLACE")
pretty_print_triplets(next(iter(pert2_dot_dataloader)), dot_model.tokenizer, num=2)
### Visualisation of perturbed data
``` {python}
# Get a single pair from each perturbation type just to visualize
pert1_batch = next(iter(pert1_dot_dataloader))
pert2_batch = next(iter(pert2_dot_dataloader))
print("solution -> answer")
pretty_print_triplets(pert1_batch, dot_model.tokenizer, num=2)
``` ```
```{python}
print("microwave -> toaster")
pretty_print_triplets(pert2_batch, dot_model.tokenizer, num=2)
```
### Select useful Perturbation
5. Apply Activation Patching
6. Select relevant perturbations
### Step 4 : Measure the impact of the perturbation on the model
```{python} ```{python}
#|echo: true
# Initialize lists to store baseline and perturbed performances for each dataloader # Initialize lists to store baseline and perturbed performances for each dataloader
all_baseline_performance = {"pert1": [], "pert2": []} all_baseline_performance = {"solution -> answer": [], "microwave -> toaster": []}
all_perturbed_performance = {"pert1": [], "pert2": []} all_perturbed_performance = {"solution -> answer": [], "microwave -> toaster": []}
# Calculate performances for each perturbation_type # Calculate performances for each perturbation_type
calculate_performance(dot_model, pert1_dot_dataloader, all_baseline_performance["pert1"], all_perturbed_performance["pert1"]) calculate_performance(dot_model, pert1_dot_dataloader, all_baseline_performance["solution -> answer"], all_perturbed_performance["solution -> answer"])
calculate_performance(dot_model, pert2_dot_dataloader, all_baseline_performance["pert2"], all_perturbed_performance["pert2"]) calculate_performance(dot_model, pert2_dot_dataloader, all_baseline_performance["microwave -> toaster"], all_perturbed_performance["microwave -> toaster"])
plot_score_dists_mult(all_baseline_performance, all_perturbed_performance, plot_type="kde") plot_score_dists_mult(all_baseline_performance, all_perturbed_performance, plot_type="kde")
``` ```
### Step 5 : Chart the sensitivity of the model to the perturbation
### Chart Model sensibility to the perturbation
7. Map the model sensibility to the perturbation
![](images/chart_append-microwave_TASB_vaswani.png)
```{python} ```{python}
# |eval: false
patching_head_outputs = [] patching_head_outputs = []
for i, batch in enumerate(pert2_dot_dataloader): for i, batch in enumerate(pert2_dot_dataloader):
queries = batch["queries"] queries = batch["queries"]
...@@ -337,14 +392,154 @@ for i, batch in enumerate(pert2_dot_dataloader): ...@@ -337,14 +392,154 @@ for i, batch in enumerate(pert2_dot_dataloader):
mean_head_outputs = torch.mean(torch.stack([tens for tens,_ in patching_head_outputs]), axis=0) mean_head_outputs = torch.mean(torch.stack([tens for tens,_ in patching_head_outputs]), axis=0)
print(mean_head_outputs)
plot_components(mean_head_outputs.detach().to("cpu").numpy()) plot_components(mean_head_outputs.detach().to("cpu").numpy())
``` ```
<!-- ![Chart of the impact of a perturbation on the components](images/ActivationPatchingAllHeadExempleMechIR.png) --> ### Finetune the Model
8. Modify sensible weights
WORK IN PROGRESS
# Experimentation
### Setup - Dataset
:::: {.columns}
::: {.column width="50%"}
- Extracted from EvalLLM2026 Challenge dataset
- French Defense dataset
- Pairs `(question, chunk of document)`
- 52 pairs, 26 questions
:::
::: {.column width="50%"}
![](images/ChallengeEvalLLM-pagewebPetit.png)
:::
::::
### Setup - Model
E5-small [@wang_text_2022]
- Constrative bi-encoder
- Pre-trained on unlabeled pairs (query, passage) CCPairs
- Finetuned with labelled pairs
- Based on MiniLM [@wang_minilmv2_2021]
- 33M params
### Setup - Mechanistic Interpretability Tools
Python libraries:
- MechIR [@parry_mechir_2025]
- TransformerLens [@nanda_transformerlens_2022]
### First Results - Domain specific language identification
- Defense-specific language is prevalent
|Word |Frequency| TF-IDF score |
|---------|---|--------------------|
|militaire|26 |0.04494102684257385 |
|drone |21 |0.04081921309724857 |
|article |13 |0.0317659447734224 |
|défense |9 |0.018444049962599192|
|emploi |7 |0.017992160135997803|
|armée |7 |0.0166576415296721 |
| ... |...|... |
### First results - Perturbations in general
:::: {.columns}
:::{.column width="50%"}
![](images/a%C3%A9rien%20-%20l%C3%A9ger_scores_paires-mainV1_exposé.png)
![](images/d%C3%A9fense%20-%20corne_scores_paires-mainV1_exposé.png)
:::
:::{.column width="50%"}
![](images/article%20-%20produit_scores_paires-mainV1_exposé.png)
![](images/protection%20-%20bouclier_scores_paires-mainV1_exposé.png)
:::
::::
### First results - Best Perturbations {.smaller}
::::{.columns}
:::{.column width="50%"}
![](images/emploi%20-%20poste_scores_paires-mainV1_exposé.png)
![](images/militaire%20-%20soldat_scores_paires-mainV1_expos%C3%A9.png)
:::
:::{.column width="50%"}
![](images/syst%C3%A8me%20-%20ensemble%20d'%C3%A9quation_scores_paires-mainV1_expos%C3%A9.png)
- needs more complex perturbation (grammar, group of words)
- not enough
. . .
LLMs target polysemy problem of models like Word2Vect
:::
::::
### Work left for the next 2,5 months
:::{.incremental}
- Identify new perturbations
- new words and synonyms
- new type of perturbation
- Chart the sensibility of the model to new perturbations
- Identify model modifications that enhance domain specific language comprehension/ use
:::
# Conclusion & Perspectives
### Conclusion
- H1 : Models have specific components dedicated to domain specific language
- H2 : This domain specific language dedicated part of the model can be charted with activation patching
- H3 : That map can be used to enhance model comprehension and robustness of the domain specific language dedicated part of the model
$\Rightarrow$ Work in Progress
### Other perturbations and linguistic structures
- subgroups of domain specific language
- temporal relationships
- negation
- expression of diachrony
- language switching
### Other use of the map
- Transfert learning
- Prunning
- Specialization of general models
## {.remerciements}
Thanks for you attention
Questions
### Perspective : Enhance a model with MechIR
TODO
### References ### References
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment